[Bugfix][MiniMax-M3] Guard sparse decode sentinel indices (#54597)
Signed-off-by: Kyle Liang <kylliang@nvidia.com> Signed-off-by: Kyle Liang <kyleliang-nv@users.noreply.github.com> Co-authored-by: Kyle Liang <kyleliang-nv@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Kyle Liang
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
972195f4f5
commit
1664c4f444
@@ -552,6 +552,21 @@ steps:
|
||||
- pytest -v -s kernels/attention/test_deepgemm_attention.py
|
||||
- pytest -v -s quantization/test_cutlass_w4a16.py
|
||||
|
||||
- label: ":nvidia: (B200) MiniMax-M3 Attention Kernels"
|
||||
key: kernels-minimax-m3-attention-b200
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: b200-k8s
|
||||
num_devices: 1
|
||||
source_file_dependencies:
|
||||
- vllm/models/minimax_m3/
|
||||
- vllm/config/cache.py
|
||||
- vllm/v1/kv_cache_interface.py
|
||||
- tests/v1/attention/utils.py
|
||||
- tests/kernels/attention/test_minimax_m3.py
|
||||
commands:
|
||||
- pytest -v -s tests/kernels/attention/test_minimax_m3.py
|
||||
|
||||
- label: ":nvidia: (B200) Kernels Shard %N"
|
||||
key: kernels-b200
|
||||
timeout_in_minutes: 30
|
||||
@@ -601,7 +616,6 @@ steps:
|
||||
- tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py
|
||||
- tests/kernels/mamba/test_gdn_prefill_cutedsl.py
|
||||
- tests/kernels/test_bf16x3_router_gemm_cutedsl.py
|
||||
- tests/kernels/attention/test_minimax_m3.py
|
||||
- tests/kernels/test_ll_bf16_gemm.py
|
||||
- tests/kernels/test_top_k_per_row.py
|
||||
- tests/kernels/quantization/test_silu_mul_nvfp4_quant.py
|
||||
@@ -647,7 +661,6 @@ steps:
|
||||
pytest -v -s tests/kernels/moe/test_flashinfer.py
|
||||
pytest -v -s tests/kernels/moe/test_trtllm_nvfp4_moe.py
|
||||
pytest -v -s tests/kernels/mamba/test_gdn_prefill_cutedsl.py
|
||||
pytest -v -s tests/kernels/attention/test_minimax_m3.py
|
||||
;;
|
||||
*)
|
||||
echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" >&2
|
||||
|
||||
@@ -2612,6 +2612,103 @@ def test_decode_sparse_attention_correctness(
|
||||
assert error.max().item() < 1.7e-2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
|
||||
@pytest.mark.parametrize("capture_graph", [False, True])
|
||||
@pytest.mark.parametrize("num_reqs", [2, 17], ids=["split-k", "single-chunk"])
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_block", [-1, 1_000_000], ids=["sentinel", "out-of-range"]
|
||||
)
|
||||
def test_decode_sparse_attention_ignores_invalid_topk(
|
||||
kv_layout: KVCacheLayout, capture_graph: bool, num_reqs: int, invalid_block: int
|
||||
):
|
||||
"""Unused top-k entries must not be dereferenced as block-table indices."""
|
||||
torch.manual_seed(0)
|
||||
decode_query_len = 4
|
||||
seq_lens = torch.full(
|
||||
(num_reqs,), decode_query_len, device="cuda", dtype=torch.int32
|
||||
)
|
||||
seq_lens[1] = 4096
|
||||
q = torch.randn(
|
||||
seq_lens.numel() * decode_query_len,
|
||||
NUM_Q_HEADS,
|
||||
HEAD_DIM,
|
||||
device="cuda",
|
||||
dtype=DTYPE,
|
||||
)
|
||||
kv_cache = _allocate_main_kv_via_contract(1, kv_layout)
|
||||
|
||||
# All logical blocks alias the single valid page. With a contiguous multi-row
|
||||
# table, row 1's raw pointer offset -1 resolves to the final element of row
|
||||
# 0. Poison that exact location so an unsafe load deterministically forms an
|
||||
# out-of-range KV-cache pointer.
|
||||
clean_block_table = torch.zeros(num_reqs, 32, device="cuda", dtype=torch.int32)
|
||||
block_table = clean_block_table.clone()
|
||||
block_table[0, -1] = 1_000_000
|
||||
topk_idx = torch.full(
|
||||
(NUM_KV_HEADS, q.shape[0], TOPK),
|
||||
-1,
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
topk_idx[..., 0] = 0
|
||||
# Request 1 has invalid entries both before and after its valid block.
|
||||
# Two requests produce empty split-K chunks; 17 put the full top-k in one
|
||||
# chunk, exercising neutral updates before and after a nonempty softmax.
|
||||
request_topk = topk_idx[:, decode_query_len : 2 * decode_query_len]
|
||||
request_topk.fill_(invalid_block)
|
||||
request_topk[..., TOPK // 2] = 0
|
||||
|
||||
# A valid-only run isolates sentinel handling from BF16-vs-FP32 rounding.
|
||||
# Include block 0 exactly once; replacing sentinels with 0 would duplicate
|
||||
# its contribution. The general correctness test covers reference accuracy.
|
||||
valid_topk = torch.zeros(
|
||||
(NUM_KV_HEADS, q.shape[0], 1), device="cuda", dtype=torch.int32
|
||||
)
|
||||
expected = torch.empty_like(q)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_cache,
|
||||
valid_topk,
|
||||
clean_block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
SM_SCALE,
|
||||
expected,
|
||||
decode_query_len,
|
||||
)
|
||||
actual = torch.empty_like(q)
|
||||
|
||||
def run_decode() -> None:
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_cache,
|
||||
topk_idx,
|
||||
block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
SM_SCALE,
|
||||
actual,
|
||||
decode_query_len,
|
||||
)
|
||||
|
||||
if capture_graph:
|
||||
# An unfixed kernel faults during capture and leaves the CUDA context
|
||||
# unusable. Run negative-control reproductions in a dedicated process.
|
||||
run_decode()
|
||||
current_platform.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run_decode()
|
||||
for _ in range(3):
|
||||
graph.replay()
|
||||
else:
|
||||
run_decode()
|
||||
current_platform.synchronize()
|
||||
|
||||
assert torch.isfinite(actual).all()
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_decode_wrong_layout_breaks_parity():
|
||||
"""Negative (AC-3/AC-5): consuming the physical HND buffer as if it were
|
||||
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
|
||||
|
||||
@@ -304,7 +304,7 @@ def _gqa_sparse_decode_kernel(
|
||||
# attention range instead of letting padded rows produce negative lengths.
|
||||
kv_len = tl.maximum(query_pos + 1, 0)
|
||||
|
||||
# Valid block count from seq_len (no sentinel): min(topk, cdiv(kv_len, blk)).
|
||||
# Upper bound on top-k entries from causal length; entries may be sentinels.
|
||||
idx_base = t_ptr + pid_kh * stride_th + pid_b * stride_tn
|
||||
num_blocks = (kv_len + BLOCK_SIZE_K - 1) // BLOCK_SIZE_K
|
||||
real_topk = tl.minimum(max_topk, num_blocks)
|
||||
@@ -334,10 +334,15 @@ def _gqa_sparse_decode_kernel(
|
||||
for _ in tl.range(chunk_start_topk, chunk_end_topk):
|
||||
blk = tl.load(cur_idx_ptr).to(tl.int32)
|
||||
cur_idx_ptr = cur_idx_ptr + stride_tk
|
||||
c = blk * BLOCK_SIZE_K
|
||||
page = tl.load(bt_row + blk).to(tl.int64)
|
||||
# topk_idx uses -1 for unused entries. Select a safe logical block
|
||||
# before dereferencing the block table; invalid and future blocks then
|
||||
# contribute a neutral online-softmax update without control flow.
|
||||
valid_blk = (blk >= 0) & (blk < num_blocks)
|
||||
safe_blk = tl.where(valid_blk, blk, 0)
|
||||
c = safe_blk * BLOCK_SIZE_K
|
||||
page = tl.load(bt_row + safe_blk).to(tl.int64)
|
||||
pos = c + off_n
|
||||
pos_mask = pos < kv_len
|
||||
pos_mask = (pos < kv_len) & valid_blk
|
||||
k = tl.load(
|
||||
kv_cache_ptr
|
||||
+ page * stride_kv_blk
|
||||
@@ -364,9 +369,11 @@ def _gqa_sparse_decode_kernel(
|
||||
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
|
||||
qk += tl.dot(q, k) * sm_scale_log2e
|
||||
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
|
||||
p = tl.exp2(qk - m_ij[:, None])
|
||||
m_safe = tl.where(valid_blk, m_ij, 0.0)
|
||||
p = tl.where(pos_mask[None, :], tl.exp2(qk - m_safe[:, None]), 0.0)
|
||||
l_ij = tl.sum(p, axis=1)
|
||||
acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None]
|
||||
alpha = tl.where(valid_blk, tl.exp2(m_i - m_safe), 1.0)
|
||||
acc_o = acc_o * alpha[:, None]
|
||||
v = tl.load(
|
||||
kv_cache_ptr
|
||||
+ page * stride_kv_blk
|
||||
@@ -390,8 +397,9 @@ def _gqa_sparse_decode_kernel(
|
||||
)
|
||||
v = (v * v_scale[:, None]).to(q.dtype)
|
||||
acc_o += tl.dot(p.to(v.dtype), v)
|
||||
m_i = m_ij
|
||||
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
|
||||
lse_ij = m_safe + tl.log2(tl.exp2(lse_i - m_safe) + l_ij)
|
||||
m_i = tl.where(valid_blk, m_safe, m_i)
|
||||
lse_i = tl.where(valid_blk, lse_ij, lse_i)
|
||||
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
Reference in New Issue
Block a user