diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index e0d6c4ce40..aa0971b4c5 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -552,6 +552,21 @@ steps: - pytest -v -s kernels/attention/test_deepgemm_attention.py - pytest -v -s quantization/test_cutlass_w4a16.py +- label: ":nvidia: (B200) MiniMax-M3 Attention Kernels" + key: kernels-minimax-m3-attention-b200 + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/" + device: b200-k8s + num_devices: 1 + source_file_dependencies: + - vllm/models/minimax_m3/ + - vllm/config/cache.py + - vllm/v1/kv_cache_interface.py + - tests/v1/attention/utils.py + - tests/kernels/attention/test_minimax_m3.py + commands: + - pytest -v -s tests/kernels/attention/test_minimax_m3.py + - label: ":nvidia: (B200) Kernels Shard %N" key: kernels-b200 timeout_in_minutes: 30 @@ -601,7 +616,6 @@ steps: - tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py - tests/kernels/mamba/test_gdn_prefill_cutedsl.py - tests/kernels/test_bf16x3_router_gemm_cutedsl.py - - tests/kernels/attention/test_minimax_m3.py - tests/kernels/test_ll_bf16_gemm.py - tests/kernels/test_top_k_per_row.py - tests/kernels/quantization/test_silu_mul_nvfp4_quant.py @@ -647,7 +661,6 @@ steps: pytest -v -s tests/kernels/moe/test_flashinfer.py pytest -v -s tests/kernels/moe/test_trtllm_nvfp4_moe.py pytest -v -s tests/kernels/mamba/test_gdn_prefill_cutedsl.py - pytest -v -s tests/kernels/attention/test_minimax_m3.py ;; *) echo "unexpected BUILDKITE_PARALLEL_JOB=$$BUILDKITE_PARALLEL_JOB" >&2 diff --git a/tests/kernels/attention/test_minimax_m3.py b/tests/kernels/attention/test_minimax_m3.py index 7f0e073093..7ee900fd3d 100644 --- a/tests/kernels/attention/test_minimax_m3.py +++ b/tests/kernels/attention/test_minimax_m3.py @@ -2612,6 +2612,103 @@ def test_decode_sparse_attention_correctness( assert error.max().item() < 1.7e-2 +@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True) +@pytest.mark.parametrize("capture_graph", [False, True]) +@pytest.mark.parametrize("num_reqs", [2, 17], ids=["split-k", "single-chunk"]) +@pytest.mark.parametrize( + "invalid_block", [-1, 1_000_000], ids=["sentinel", "out-of-range"] +) +def test_decode_sparse_attention_ignores_invalid_topk( + kv_layout: KVCacheLayout, capture_graph: bool, num_reqs: int, invalid_block: int +): + """Unused top-k entries must not be dereferenced as block-table indices.""" + torch.manual_seed(0) + decode_query_len = 4 + seq_lens = torch.full( + (num_reqs,), decode_query_len, device="cuda", dtype=torch.int32 + ) + seq_lens[1] = 4096 + q = torch.randn( + seq_lens.numel() * decode_query_len, + NUM_Q_HEADS, + HEAD_DIM, + device="cuda", + dtype=DTYPE, + ) + kv_cache = _allocate_main_kv_via_contract(1, kv_layout) + + # All logical blocks alias the single valid page. With a contiguous multi-row + # table, row 1's raw pointer offset -1 resolves to the final element of row + # 0. Poison that exact location so an unsafe load deterministically forms an + # out-of-range KV-cache pointer. + clean_block_table = torch.zeros(num_reqs, 32, device="cuda", dtype=torch.int32) + block_table = clean_block_table.clone() + block_table[0, -1] = 1_000_000 + topk_idx = torch.full( + (NUM_KV_HEADS, q.shape[0], TOPK), + -1, + device="cuda", + dtype=torch.int32, + ) + topk_idx[..., 0] = 0 + # Request 1 has invalid entries both before and after its valid block. + # Two requests produce empty split-K chunks; 17 put the full top-k in one + # chunk, exercising neutral updates before and after a nonempty softmax. + request_topk = topk_idx[:, decode_query_len : 2 * decode_query_len] + request_topk.fill_(invalid_block) + request_topk[..., TOPK // 2] = 0 + + # A valid-only run isolates sentinel handling from BF16-vs-FP32 rounding. + # Include block 0 exactly once; replacing sentinels with 0 would duplicate + # its contribution. The general correctness test covers reference accuracy. + valid_topk = torch.zeros( + (NUM_KV_HEADS, q.shape[0], 1), device="cuda", dtype=torch.int32 + ) + expected = torch.empty_like(q) + minimax_m3_sparse_attn_decode( + q, + kv_cache, + valid_topk, + clean_block_table, + seq_lens, + NUM_KV_HEADS, + SM_SCALE, + expected, + decode_query_len, + ) + actual = torch.empty_like(q) + + def run_decode() -> None: + minimax_m3_sparse_attn_decode( + q, + kv_cache, + topk_idx, + block_table, + seq_lens, + NUM_KV_HEADS, + SM_SCALE, + actual, + decode_query_len, + ) + + if capture_graph: + # An unfixed kernel faults during capture and leaves the CUDA context + # unusable. Run negative-control reproductions in a dedicated process. + run_decode() + current_platform.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run_decode() + for _ in range(3): + graph.replay() + else: + run_decode() + current_platform.synchronize() + + assert torch.isfinite(actual).all() + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + def test_decode_wrong_layout_breaks_parity(): """Negative (AC-3/AC-5): consuming the physical HND buffer as if it were already contiguous-NHD (i.e. skipping the allocator's inverse permute) diff --git a/vllm/models/minimax_m3/common/ops/sparse_attn.py b/vllm/models/minimax_m3/common/ops/sparse_attn.py index 8dfd6e0c19..6426115ae3 100644 --- a/vllm/models/minimax_m3/common/ops/sparse_attn.py +++ b/vllm/models/minimax_m3/common/ops/sparse_attn.py @@ -304,7 +304,7 @@ def _gqa_sparse_decode_kernel( # attention range instead of letting padded rows produce negative lengths. kv_len = tl.maximum(query_pos + 1, 0) - # Valid block count from seq_len (no sentinel): min(topk, cdiv(kv_len, blk)). + # Upper bound on top-k entries from causal length; entries may be sentinels. idx_base = t_ptr + pid_kh * stride_th + pid_b * stride_tn num_blocks = (kv_len + BLOCK_SIZE_K - 1) // BLOCK_SIZE_K real_topk = tl.minimum(max_topk, num_blocks) @@ -334,10 +334,15 @@ def _gqa_sparse_decode_kernel( for _ in tl.range(chunk_start_topk, chunk_end_topk): blk = tl.load(cur_idx_ptr).to(tl.int32) cur_idx_ptr = cur_idx_ptr + stride_tk - c = blk * BLOCK_SIZE_K - page = tl.load(bt_row + blk).to(tl.int64) + # topk_idx uses -1 for unused entries. Select a safe logical block + # before dereferencing the block table; invalid and future blocks then + # contribute a neutral online-softmax update without control flow. + valid_blk = (blk >= 0) & (blk < num_blocks) + safe_blk = tl.where(valid_blk, blk, 0) + c = safe_blk * BLOCK_SIZE_K + page = tl.load(bt_row + safe_blk).to(tl.int64) pos = c + off_n - pos_mask = pos < kv_len + pos_mask = (pos < kv_len) & valid_blk k = tl.load( kv_cache_ptr + page * stride_kv_blk @@ -364,9 +369,11 @@ def _gqa_sparse_decode_kernel( qk += tl.where(pos_mask[None, :], 0, float("-inf")) qk += tl.dot(q, k) * sm_scale_log2e m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) - p = tl.exp2(qk - m_ij[:, None]) + m_safe = tl.where(valid_blk, m_ij, 0.0) + p = tl.where(pos_mask[None, :], tl.exp2(qk - m_safe[:, None]), 0.0) l_ij = tl.sum(p, axis=1) - acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None] + alpha = tl.where(valid_blk, tl.exp2(m_i - m_safe), 1.0) + acc_o = acc_o * alpha[:, None] v = tl.load( kv_cache_ptr + page * stride_kv_blk @@ -390,8 +397,9 @@ def _gqa_sparse_decode_kernel( ) v = (v * v_scale[:, None]).to(q.dtype) acc_o += tl.dot(p.to(v.dtype), v) - m_i = m_ij - lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij) + lse_ij = m_safe + tl.log2(tl.exp2(lse_i - m_safe) + l_ij) + m_i = tl.where(valid_blk, m_safe, m_i) + lse_i = tl.where(valid_blk, lse_ij, lse_i) if USE_PDL: tl.extra.cuda.gdc_launch_dependents()