[Minimax-M3] Using tok_sparse_select from MSA instead of triton kernels (#47502)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
0303f37a54
commit
9021589498
@@ -17,7 +17,7 @@ else()
|
||||
FetchContent_Declare(
|
||||
fmha_sm100
|
||||
GIT_REPOSITORY https://github.com/vllm-project/MSA.git
|
||||
GIT_TAG fee783153f3efe57e3e933c5cb7e267a7cebcfb5
|
||||
GIT_TAG 2e63ec37a0fc29bc20f39cd1a52e0f5affc33a73
|
||||
GIT_PROGRESS TRUE
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
|
||||
@@ -150,7 +150,7 @@ def _reference_index_topk(
|
||||
num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
|
||||
pages = block_table[req_id, :num_blocks]
|
||||
k = index_kv_cache[pages].reshape(num_blocks * BLOCK_SIZE, -1)
|
||||
score = torch.einsum("qhd,kd->hqk", q.float(), k.float()) * sm_scale
|
||||
score = sm_scale * torch.einsum("qhd,kd->hqk", q.float(), k.float())
|
||||
|
||||
q_pos = prefix_len + torch.arange(q_len, device=idx_q.device)
|
||||
k_pos = torch.arange(k.shape[0], device=idx_q.device)
|
||||
@@ -621,11 +621,12 @@ def test_decode_index_topk_fp8(num_idx_heads: int):
|
||||
init_blocks=init_blocks,
|
||||
local_blocks=local_blocks,
|
||||
num_kv_heads=num_idx_heads,
|
||||
sm_scale=head_dim**-0.5,
|
||||
decode_query_len=decode_query_len,
|
||||
max_decode_query_len=decode_query_len,
|
||||
)
|
||||
# Reference from the DEQUANTIZED fp8 values (the kernel computes the fp8 QK
|
||||
# in fp32, so it must match an fp32 matmul of the same e4m3 values).
|
||||
# in fp32 with no scaling, so it must match an unscaled fp32 matmul of the
|
||||
# same e4m3 values).
|
||||
expected = _reference_index_topk(
|
||||
idx_q.float(),
|
||||
index_kv_cache.float(),
|
||||
@@ -636,7 +637,6 @@ def test_decode_index_topk_fp8(num_idx_heads: int):
|
||||
topk,
|
||||
init_blocks,
|
||||
local_blocks,
|
||||
head_dim**-0.5,
|
||||
)
|
||||
_assert_topk_indices_equal_unordered(actual, expected)
|
||||
|
||||
|
||||
@@ -256,6 +256,13 @@ class MiniMaxM3IndexerMetadataBuilder(
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
# Stable per-token causal page-count buffer for decode cudagraph replays
|
||||
# (consumed by the MSA top-k path's sparse_topk_select num_valid_pages).
|
||||
self.num_valid_pages_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM3IndexerTritonMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
from .index_topk import (
|
||||
minimax_m3_index_decode,
|
||||
minimax_m3_index_decode_score,
|
||||
minimax_m3_index_score,
|
||||
minimax_m3_index_topk,
|
||||
)
|
||||
@@ -11,6 +12,7 @@ from .sparse_attn import minimax_m3_sparse_attn, minimax_m3_sparse_attn_decode
|
||||
|
||||
__all__ = [
|
||||
"minimax_m3_index_decode",
|
||||
"minimax_m3_index_decode_score",
|
||||
"minimax_m3_index_score",
|
||||
"minimax_m3_index_topk",
|
||||
"minimax_m3_sparse_attn",
|
||||
|
||||
@@ -757,25 +757,26 @@ def minimax_m3_index_topk(
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def minimax_m3_index_decode(
|
||||
def minimax_m3_index_decode_score(
|
||||
idx_q: torch.Tensor, # [total_q, num_idx_heads, head_dim]
|
||||
index_kv_cache: torch.Tensor, # [num_blocks, 128, head_dim]
|
||||
block_table: torch.Tensor, # [num_reqs, max_blocks]
|
||||
seq_lens: torch.Tensor, # [num_reqs] int32
|
||||
max_seq_len: int,
|
||||
topk: int,
|
||||
init_blocks: int,
|
||||
local_blocks: int,
|
||||
num_kv_heads: int,
|
||||
decode_query_len: int,
|
||||
max_decode_query_len: int,
|
||||
out: torch.Tensor | None = None,
|
||||
score_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Decode index block-score + top-k, both split-K (cudagraph-safe).
|
||||
"""Decode index block-score (split-K, cudagraph-safe); no top-k.
|
||||
|
||||
Returns topk_idx [num_kv_heads, total_q, topk] (0-indexed block ids, -1 pad).
|
||||
When ``out`` ([num_kv_heads, >=total_q, topk]) is given, writes into
|
||||
``out[:, :total_q, :]`` (stable address for cudagraph) instead of allocating.
|
||||
Returns score [num_kv_heads, total_q, >=max_block] (fp32; init/local blocks
|
||||
forced to 1e30/1e29). When ``score_out`` is given the scores are written into
|
||||
it (read/written by strides, so a transposed view of a unified buffer is
|
||||
accepted) instead of a fresh tensor -- used to share a unified score buffer
|
||||
with the prefill side and run a single top-k over both.
|
||||
"""
|
||||
total_q, num_idx_heads, head_dim = idx_q.shape
|
||||
assert num_idx_heads == num_kv_heads, (
|
||||
@@ -783,7 +784,6 @@ def minimax_m3_index_decode(
|
||||
)
|
||||
assert decode_query_len <= max_decode_query_len
|
||||
assert total_q == seq_lens.shape[0] * decode_query_len
|
||||
batch = total_q
|
||||
max_block = triton.cdiv(max_seq_len, SPARSE_BLOCK_SIZE)
|
||||
use_pdl = current_platform.is_arch_support_pdl()
|
||||
# `launch_pdl` is a Triton runtime kwarg only some backends accept (CUDA
|
||||
@@ -800,13 +800,16 @@ def minimax_m3_index_decode(
|
||||
if num_idx_heads > 1 and max_decode_query_len > 1:
|
||||
score_kwargs.update({"num_warps": 4, "num_stages": 2})
|
||||
|
||||
# Keep score strides 16-divisible to avoid Triton recompiles.
|
||||
score_block_stride = round_up(max_block, 16)
|
||||
score = torch.empty(
|
||||
(num_idx_heads, total_q, score_block_stride),
|
||||
dtype=torch.float32,
|
||||
device=idx_q.device,
|
||||
)
|
||||
if score_out is not None:
|
||||
score = score_out
|
||||
else:
|
||||
# Keep score strides 16-divisible to avoid Triton recompiles.
|
||||
score_block_stride = round_up(max_block, 16)
|
||||
score = torch.empty(
|
||||
(num_idx_heads, total_q, score_block_stride),
|
||||
dtype=torch.float32,
|
||||
device=idx_q.device,
|
||||
)
|
||||
# split-K over seq blocks; chunk count depends only on shape constants so
|
||||
# the grid is fixed within a cuda graph.
|
||||
TARGET_GRID = 512
|
||||
@@ -848,6 +851,55 @@ def minimax_m3_index_decode(
|
||||
USE_PDL=use_pdl,
|
||||
**score_kwargs,
|
||||
)
|
||||
return score
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def minimax_m3_index_decode(
|
||||
idx_q: torch.Tensor, # [total_q, num_idx_heads, head_dim]
|
||||
index_kv_cache: torch.Tensor, # [num_blocks, 128, head_dim]
|
||||
block_table: torch.Tensor, # [num_reqs, max_blocks]
|
||||
seq_lens: torch.Tensor, # [num_reqs] int32
|
||||
max_seq_len: int,
|
||||
topk: int,
|
||||
init_blocks: int,
|
||||
local_blocks: int,
|
||||
num_kv_heads: int,
|
||||
decode_query_len: int,
|
||||
max_decode_query_len: int,
|
||||
out: torch.Tensor | None = None,
|
||||
score_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Decode index block-score + top-k, both split-K (cudagraph-safe).
|
||||
|
||||
Returns topk_idx [num_kv_heads, total_q, topk] (0-indexed block ids, -1 pad).
|
||||
When ``out`` ([num_kv_heads, >=total_q, topk]) is given, writes into
|
||||
``out[:, :total_q, :]`` (stable address for cudagraph) instead of allocating.
|
||||
When ``score_out`` ([num_kv_heads, total_q, >=max_block]) is given, the block
|
||||
scores are written into it (read back by the top-k) instead of a fresh
|
||||
tensor -- used to share a unified score buffer with the prefill side. Reads
|
||||
via strides, so a transposed view of a block-major buffer is accepted.
|
||||
"""
|
||||
total_q, num_idx_heads, _ = idx_q.shape
|
||||
batch = total_q
|
||||
max_block = triton.cdiv(max_seq_len, SPARSE_BLOCK_SIZE)
|
||||
use_pdl = current_platform.is_arch_support_pdl()
|
||||
pdl_kwargs: dict[str, bool | int] = {}
|
||||
if use_pdl:
|
||||
pdl_kwargs.update({"launch_pdl": True})
|
||||
score = minimax_m3_index_decode_score(
|
||||
idx_q,
|
||||
index_kv_cache,
|
||||
block_table,
|
||||
seq_lens,
|
||||
max_seq_len,
|
||||
init_blocks,
|
||||
local_blocks,
|
||||
num_kv_heads,
|
||||
decode_query_len,
|
||||
max_decode_query_len,
|
||||
score_out=score_out,
|
||||
)
|
||||
|
||||
if out is not None:
|
||||
topk_idx = out[:, :total_q, :]
|
||||
|
||||
@@ -43,7 +43,6 @@ _FP8_DTYPES = (
|
||||
{
|
||||
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
||||
"BLOCK_SIZE_H": lambda args: triton.next_power_of_2(args["gqa_group_size"]),
|
||||
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
|
||||
"BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"]
|
||||
* triton.next_power_of_2(args["gqa_group_size"]),
|
||||
}
|
||||
@@ -84,7 +83,6 @@ def _gqa_sparse_fwd_kernel(
|
||||
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
|
||||
BLOCK_SIZE_D: tl.constexpr,
|
||||
BLOCK_SIZE_H: tl.constexpr,
|
||||
BLOCK_SIZE_T: tl.constexpr,
|
||||
BLOCK_SIZE_QH: tl.constexpr,
|
||||
USE_FP8: tl.constexpr, # fp8 KV cache: dequantize K/V to q.dtype on load
|
||||
):
|
||||
@@ -109,9 +107,10 @@ def _gqa_sparse_fwd_kernel(
|
||||
for j in range(real_q_loop):
|
||||
pid_q_j = pid_q * num_q_loop + j
|
||||
t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th
|
||||
off_t = tl.arange(0, BLOCK_SIZE_T)
|
||||
topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1)
|
||||
real_topk = tl.sum((topk_idx >= 0).to(tl.int32), axis=0)
|
||||
# Valid block count from seq position (no sentinel): block_size_q == 1.
|
||||
q_abs = prefix_len + pid_q_j * BLOCK_SIZE_Q
|
||||
valid_blocks = (q_abs + BLOCK_SIZE_K) // BLOCK_SIZE_K
|
||||
real_topk = tl.minimum(max_topk, valid_blocks)
|
||||
q_ptrs = tl.make_block_ptr(
|
||||
base=q_ptr + q_start * stride_qn + pid_h * stride_qh,
|
||||
shape=(q_len, gqa_group_size, head_dim),
|
||||
@@ -202,7 +201,6 @@ def _gqa_sparse_fwd_kernel(
|
||||
16, triton.next_power_of_2(args["gqa_group_size"])
|
||||
),
|
||||
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
||||
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
|
||||
}
|
||||
)
|
||||
@triton.jit(do_not_specialize=["decode_query_len"])
|
||||
@@ -243,7 +241,6 @@ def _gqa_sparse_decode_kernel(
|
||||
NUM_TOPK_CHUNKS: tl.constexpr,
|
||||
BLOCK_SIZE_H: tl.constexpr,
|
||||
BLOCK_SIZE_D: tl.constexpr,
|
||||
BLOCK_SIZE_T: tl.constexpr,
|
||||
USE_FP8: tl.constexpr, # fp8 KV cache: dequantize K/V to q.dtype on load
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
@@ -268,11 +265,10 @@ def _gqa_sparse_decode_kernel(
|
||||
# attention range instead of letting padded rows produce negative lengths.
|
||||
kv_len = tl.maximum(query_pos + 1, 0)
|
||||
|
||||
# number of valid (non-padded) selected blocks for this query token
|
||||
off_t = tl.arange(0, BLOCK_SIZE_T)
|
||||
# Valid block count from seq_len (no sentinel): min(topk, cdiv(kv_len, blk)).
|
||||
idx_base = t_ptr + pid_kh * stride_th + pid_b * stride_tn
|
||||
topk_idx = tl.load(idx_base + off_t * stride_tk, mask=off_t < max_topk, other=-1)
|
||||
real_topk = tl.sum((topk_idx >= 0).to(tl.int32), axis=0)
|
||||
num_blocks = (kv_len + BLOCK_SIZE_K - 1) // BLOCK_SIZE_K
|
||||
real_topk = tl.minimum(max_topk, num_blocks)
|
||||
chunk_end_topk = tl.minimum(chunk_end_compiletime, real_topk)
|
||||
|
||||
off_n = tl.arange(0, BLOCK_SIZE_K)
|
||||
|
||||
@@ -2,16 +2,21 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MSA (SM100/Blackwell) indexer impl for MiniMax M3.
|
||||
|
||||
Prefill scores with ``fmha_sm100``'s score-only (``OnlyScore``) path then selects
|
||||
top-k blocks with the Triton ``minimax_m3_index_topk`` kernel -- fmha is much
|
||||
faster than Triton for the wide prefill score (benchmarked ~3-5x).
|
||||
Both sides write block scores into one unified token-major buffer
|
||||
``[total_q, H, max_k_tiles]``, then a single ``fmha_sm100.sparse_topk_select``
|
||||
selects the top-k blocks for the whole batch (decode ``[:nd]`` + prefill
|
||||
``[nd:]``) into the shared ``topk_indices_buffer``. It bounds each row by its
|
||||
causal page count and force-includes the init/local blocks, so the unwritten
|
||||
tail of the buffer is pre-filled with ``-inf``.
|
||||
|
||||
Decode uses the Triton fused ``minimax_m3_index_decode`` (the same kernel the
|
||||
Triton indexer impl uses): for q_len==1 it is a purpose-built vector x matrix
|
||||
score (no wasted tensor-core tiles) with a 256-way split-K and a fused split-K
|
||||
top-k, which beats fmha's OnlyScore (wasted MMA on a single query, 64-split cap)
|
||||
by ~1.1-3.7x. It is cudagraph-safe by construction (shape-constant split grids)
|
||||
and writes the shared ``topk_indices_buffer`` via ``out=``.
|
||||
Prefill scores with ``fmha_sm100``'s score-only (``OnlyScore``) path (much faster
|
||||
than Triton for the wide prefill score, benchmarked ~3-5x), writing its
|
||||
``max_score`` straight into the buffer's prefill region (stride-aware, no copy).
|
||||
|
||||
Decode scores with the Triton split-K ``minimax_m3_index_decode_score`` (a
|
||||
purpose-built vector x matrix score, no wasted tensor-core tiles, 256-way
|
||||
split-K, cudagraph-safe by shape-constant grids), writing into the decode
|
||||
region. Its tuning heuristics are kept; only the top-k is shared with prefill.
|
||||
|
||||
``fmha_sm100`` imports are function-local so this module is import-safe on
|
||||
AMD / non-SM100.
|
||||
@@ -22,6 +27,7 @@ from typing import ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.minimax_m3.common.indexer import (
|
||||
MiniMaxM3IndexerBackend,
|
||||
@@ -31,8 +37,7 @@ from vllm.models.minimax_m3.common.indexer import (
|
||||
MiniMaxM3IndexerMetadataBuilder,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.ops.index_topk import (
|
||||
minimax_m3_index_decode,
|
||||
minimax_m3_index_topk,
|
||||
minimax_m3_index_decode_score,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
@@ -40,10 +45,22 @@ from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
# Page size == sparse block size == index-K block; fmha tile id == M3 block id.
|
||||
PAGE_SIZE = 128
|
||||
|
||||
# Fill for unwritten score tiles: -inf so they never win the top-k (score kernels
|
||||
# only write causally-valid blocks).
|
||||
_SCORE_SENTINEL = float("-inf")
|
||||
|
||||
# Tile (KV-block) dim of the unified score buffer, hardcoded as a cudagraph
|
||||
# capture-time constant so the decode score kernel's buffer shape is frozen
|
||||
# across replays. 8192 tiles == 1M tokens of context; -inf padding +
|
||||
# num_valid_pages bound each row to its causal range, so shorter replays reuse
|
||||
# the same buffer safely.
|
||||
MAX_K_TILES = 8192
|
||||
|
||||
|
||||
class MiniMaxM3IndexerMSABackend(MiniMaxM3IndexerBackend):
|
||||
"""Indexer side-cache backend selecting the MSA builder."""
|
||||
@@ -71,6 +88,25 @@ class MiniMaxM3IndexerMSAMetadata(MiniMaxM3IndexerMetadata):
|
||||
(the base ``prefill`` field is unused on this path)."""
|
||||
|
||||
prefill_msa: MiniMaxM3IndexerMSAPrefillMetadata | None = None
|
||||
# Per-forward view (``[:num_tokens]``) of the builder's persistent unified
|
||||
# score buffer ``[total_q, H, MAX_K_TILES]``, shared by decode and prefill
|
||||
# and reused across all layers. Pre-filled with the -inf sentinel in
|
||||
# ``build()``; each layer overwrites its valid tiles before its own top-k.
|
||||
unified_scores: torch.Tensor | None = None
|
||||
# Tile (KV-block) dim of the unified score buffer (== ``MAX_K_TILES``).
|
||||
# Forced as the fmha plan's ``max_k_tiles`` so prefill writes its max_score
|
||||
# straight into the shared buffer.
|
||||
max_k_tiles: int = 0
|
||||
# Batch-wide inputs for the single top-k over the unified buffer (decode +
|
||||
# prefill in one call). ``cu_seqlens_q`` is the per-request query-start
|
||||
# offsets (== query_start_loc, a stable view for cudagraph); ``prefix_lens``
|
||||
# is the per-request context length (== context_lens).
|
||||
topk_cu_seqlens_q: torch.Tensor | None = None
|
||||
topk_prefix_lens: torch.Tensor | None = None
|
||||
topk_max_query_len: int = 0
|
||||
# Per-token causal page count cdiv(seq_pos+1, PAGE_SIZE), [total_q] int32:
|
||||
# drives sparse_topk_select force_end_blocks + -1 out-of-range clamp.
|
||||
topk_num_valid_pages: torch.Tensor | None = None
|
||||
|
||||
|
||||
class MiniMaxM3IndexerMSAMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
|
||||
@@ -79,6 +115,28 @@ class MiniMaxM3IndexerMSAMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
|
||||
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
# Persistent unified score buffer [T, H, MAX_K_TILES] shared by all
|
||||
# indexer layers and reused across forwards. Stable address (required
|
||||
# for the captured decode path) + fixed tile dim so the decode score
|
||||
# kernel's shape is frozen at capture. Filled with -inf per forward in
|
||||
# build(); the valid/padding partition is a per-forward constant, so
|
||||
# every layer overwrites the same valid tiles before its own top-k.
|
||||
self.unified_scores_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
self.num_index_heads,
|
||||
MAX_K_TILES,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -107,6 +165,19 @@ class MiniMaxM3IndexerMSAMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
|
||||
common_attn_metadata.compute_num_computed_tokens(), non_blocking=True
|
||||
)
|
||||
|
||||
# Per-token causal page count for the top-k, into the stable cg buffer.
|
||||
positions = common_attn_metadata.positions
|
||||
assert positions is not None
|
||||
num_valid_pages = self.num_valid_pages_buffer[:num_tokens]
|
||||
num_valid_pages.copy_(positions[:num_tokens] // PAGE_SIZE + 1)
|
||||
|
||||
# Unified score buffer: a per-forward view of the persistent buffer,
|
||||
# reset to the -inf sentinel once here and shared by every layer (the
|
||||
# tile dim is the capture-time constant MAX_K_TILES).
|
||||
max_k_tiles = MAX_K_TILES
|
||||
unified_scores = self.unified_scores_buffer[:num_tokens]
|
||||
unified_scores.fill_(_SCORE_SENTINEL)
|
||||
|
||||
decode: MiniMaxM3IndexerDecodeMetadata | None = None
|
||||
if num_decodes > 0:
|
||||
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
@@ -148,6 +219,11 @@ class MiniMaxM3IndexerMSAMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
|
||||
causal=True,
|
||||
num_kv_splits=1,
|
||||
)
|
||||
# Force the plan's tile dim to the unified buffer's so prefill writes
|
||||
# its max_score straight into unified[:, :, nd:] (the stride-aware
|
||||
# binding shape-matches the tile dim exactly). max_k_tiles >= the
|
||||
# plan's natural value, so the extra tiles are simply never written.
|
||||
plan["max_k_tiles"] = max_k_tiles
|
||||
cols = torch.arange(block_table.shape[1], device=block_table.device)
|
||||
valid = cols[None, :] < nvp[lo:hi].to(block_table.device)[:, None]
|
||||
prefill = MiniMaxM3IndexerMSAPrefillMetadata(
|
||||
@@ -171,6 +247,12 @@ class MiniMaxM3IndexerMSAMetadataBuilder(MiniMaxM3IndexerMetadataBuilder):
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
decode=decode,
|
||||
prefill_msa=prefill,
|
||||
unified_scores=unified_scores,
|
||||
max_k_tiles=max_k_tiles,
|
||||
topk_cu_seqlens_q=query_start_loc[: num_reqs + 1],
|
||||
topk_prefix_lens=context_lens,
|
||||
topk_max_query_len=common_attn_metadata.max_query_len,
|
||||
topk_num_valid_pages=num_valid_pages,
|
||||
)
|
||||
|
||||
|
||||
@@ -195,36 +277,49 @@ class MiniMaxM3IndexerMSAImpl(MiniMaxM3IndexerImpl):
|
||||
-1, self.num_index_heads, self.index_head_dim
|
||||
)
|
||||
kv = self.index_cache.kv_cache
|
||||
# Both sides write into the single shared persistent topk_indices_buffer:
|
||||
# decode at [:, :nd], prefill at [:, nd:] (each kernel writes [:, :total_q]).
|
||||
# Shared persistent top-k output buffer; the unified top-k below writes
|
||||
# the selected block ids into buf[:num_tokens].
|
||||
buf = self.topk_indices_buffer
|
||||
assert buf is not None
|
||||
|
||||
decode_topk: torch.Tensor | None = None
|
||||
# Unified token-major score buffer [total_q, H, MAX_K_TILES]: the tile
|
||||
# dim is innermost/contiguous, so both fmha writes (native [T,H,K]) and
|
||||
# the block-iterating top-k reads hit contiguous tiles. Each side gets a
|
||||
# contiguous slice on dim 0: decode [:nd], prefill [nd:]; the kernels
|
||||
# read/write by strides. The builder allocates it once (persistent,
|
||||
# shared by all layers) and resets it to the sentinel each forward, so
|
||||
# the top-k never picks an unwritten tile.
|
||||
unified_scores = md.unified_scores
|
||||
assert unified_scores is not None
|
||||
|
||||
# Decode scores -> unified[:nd] (transposed [H, nd, MK] view; the kernel
|
||||
# writes by strides). Top-k is deferred to the single unified call below.
|
||||
if md.decode is not None:
|
||||
d = md.decode
|
||||
decode_topk = minimax_m3_index_decode(
|
||||
minimax_m3_index_decode_score(
|
||||
index_q[:nd],
|
||||
kv,
|
||||
d.block_table,
|
||||
d.seq_lens,
|
||||
d.max_seq_len,
|
||||
self.topk_blocks,
|
||||
self.init_blocks,
|
||||
self.local_blocks,
|
||||
self.num_kv_heads,
|
||||
d.decode_query_len,
|
||||
d.max_decode_query_len,
|
||||
out=buf,
|
||||
score_out=unified_scores[:nd].transpose(0, 1),
|
||||
)
|
||||
|
||||
prefill_topk: torch.Tensor | None = None
|
||||
if md.prefill_msa is not None:
|
||||
from vllm.third_party.fmha_sm100.api import _fmha_sm100
|
||||
|
||||
p = md.prefill_msa
|
||||
# Index-K cache (num_blocks, 128, D) -> paged MQA (num_blocks,1,128,D).
|
||||
k_pages = kv.view(kv.shape[0], 1, PAGE_SIZE, self.index_head_dim)
|
||||
_, max_score = _fmha_sm100(
|
||||
# fmha writes its max_score natively as [nnz_p, H, max_k_tiles] into
|
||||
# the prefill region (stride-aware; plan max_k_tiles forced to
|
||||
# md.max_k_tiles so the shape matches exactly -> no copy).
|
||||
_fmha_sm100(
|
||||
index_q[nd:],
|
||||
k_pages,
|
||||
k_pages, # V placeholder; not read in OnlyScore
|
||||
@@ -233,19 +328,24 @@ class MiniMaxM3IndexerMSAImpl(MiniMaxM3IndexerImpl):
|
||||
output_o=False,
|
||||
output_maxscore=True,
|
||||
sm_scale=self.scale,
|
||||
)
|
||||
# Triton top-k wants [num_index_heads, num_tokens, max_block]; the
|
||||
# transpose is a strided view (the kernel reads via strides).
|
||||
out = buf[:, nd:, :] if buf is not None else None
|
||||
prefill_topk = minimax_m3_index_topk(
|
||||
max_score.transpose(1, 2),
|
||||
p.cu_seqlens_q,
|
||||
p.prefix_lens,
|
||||
p.max_query_len,
|
||||
self.topk_blocks,
|
||||
self.init_blocks,
|
||||
self.local_blocks,
|
||||
out=out,
|
||||
max_score=unified_scores[nd:],
|
||||
)
|
||||
|
||||
return decode_topk, prefill_topk
|
||||
# Single top-k over the unified buffer via fmha_sm100 sparse_topk_select
|
||||
# (THK, no transpose) into ``buf``. num_valid_pages drives force_end_blocks
|
||||
# (always keep each token's local block; fmha OnlyScore won't) + the -1
|
||||
# out-of-range clamp.
|
||||
from vllm.third_party.fmha_sm100.api import sparse_topk_select
|
||||
|
||||
sparse_topk_select(
|
||||
unified_scores,
|
||||
self.topk_blocks,
|
||||
num_valid_pages=md.topk_num_valid_pages,
|
||||
force_begin_blocks=self.init_blocks,
|
||||
force_end_blocks=self.local_blocks,
|
||||
output=buf[:num_tokens],
|
||||
max_score_layout="THK",
|
||||
)
|
||||
|
||||
# The attend reads ``buf`` directly; this return is vestigial.
|
||||
return None, None
|
||||
|
||||
@@ -769,8 +769,9 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
|
||||
# Reserved top-k indices buffer shared by all sparse-attention indexer
|
||||
# layers (mirrors DeepseekV4); kept at a stable address so the indexer's
|
||||
# top-k output survives cudagraph capture/replay. Shape matches the
|
||||
# per-head index top-k output [num_index_heads, total_q, topk].
|
||||
# top-k output survives cudagraph capture/replay. Token-major
|
||||
# [total_q, num_index_heads, topk] so the indexer writes its native
|
||||
# [token, head, topk] top-k; the attend transposes to [H, tokens, topk].
|
||||
sparse_cfg = getattr(config, "sparse_attention_config", None)
|
||||
if sparse_cfg is not None:
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
@@ -780,8 +781,8 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin):
|
||||
max_num_batched_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
padded_num_tokens = (max_num_batched_tokens + 3) // 4 * 4
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
num_index_heads,
|
||||
padded_num_tokens,
|
||||
num_index_heads,
|
||||
sparse_cfg["sparse_topk_blocks"],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
@@ -39,7 +39,8 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
|
||||
nd = main_md.num_decode_tokens
|
||||
num_tokens = main_md.num_actual_tokens
|
||||
# Indexer top-k from the shared buffer: decode [:, :nd], prefill [:, nd:].
|
||||
# Indexer top-k from the shared token-major buffer [total_q, H, MK]; the
|
||||
# kernels want [H, tokens, MK], so slice tokens on dim 0 then transpose.
|
||||
topk = layer.topk_indices_buffer # type: ignore[attr-defined]
|
||||
assert topk is not None
|
||||
hd = self.head_size
|
||||
@@ -56,7 +57,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q[:nd],
|
||||
kv_cache,
|
||||
topk[:, :nd, :],
|
||||
topk[:nd].transpose(0, 1),
|
||||
d.block_table,
|
||||
d.seq_lens,
|
||||
self.num_kv_heads,
|
||||
@@ -74,7 +75,9 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
|
||||
p = main_md.prefill
|
||||
assert p is not None
|
||||
prefill_topk = topk[:, nd:num_tokens, :]
|
||||
# [H, prefill, MK] transposed view; build_k2q_csr consumes the
|
||||
# strided view directly (topK stays innermost-contiguous).
|
||||
prefill_topk = topk[nd:num_tokens].transpose(0, 1)
|
||||
qp = q[nd:]
|
||||
k_cache = kv_cache[:, 0].transpose(1, 2)
|
||||
v_cache = kv_cache[:, 1].transpose(1, 2)
|
||||
|
||||
Reference in New Issue
Block a user