From 67f9046e4a12d939ffad0abc917585a00ef5b42d Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Thu, 16 Jul 2026 18:44:04 -0400 Subject: [PATCH] [Bugfix] Sparse MLA: enable fp8_ds_mla dense prefill (#48642) Signed-off-by: Matthew Bonanni Co-authored-by: OpenAI Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- benchmarks/kernels/bench_cp_gather_fp8.py | 7 +- csrc/cache.h | 3 +- csrc/libtorch_stable/cache_kernels.cu | 28 +- csrc/libtorch_stable/ops.h | 4 +- csrc/libtorch_stable/torch_bindings.cpp | 4 +- tests/kernels/test_cp_gather_fp8.py | 76 ++++- tests/v1/attention/test_mla_backends.py | 30 +- .../v1/attention/test_sparse_mla_backends.py | 322 +++++++++++++++++- vllm/_custom_ops.py | 6 +- .../layers/attention/mla_attention.py | 31 +- .../layers/attention/sparse_mla_attention.py | 4 +- .../attention/backends/mla/flashmla_sparse.py | 81 +++-- .../v1/attention/backends/mla/sparse_utils.py | 5 + 13 files changed, 516 insertions(+), 85 deletions(-) diff --git a/benchmarks/kernels/bench_cp_gather_fp8.py b/benchmarks/kernels/bench_cp_gather_fp8.py index 19fc84c4df..ca76597a1f 100644 --- a/benchmarks/kernels/bench_cp_gather_fp8.py +++ b/benchmarks/kernels/bench_cp_gather_fp8.py @@ -69,12 +69,11 @@ def make_inputs(total_tokens, num_reqs, block_size): # Output workspace dst = torch.zeros(total_tokens, HEAD_DIM, dtype=torch.bfloat16, device="cuda") - seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda") workspace_starts_t = torch.tensor( workspace_starts, dtype=torch.int32, device="cuda" ) - return cache, dst, block_table, seq_lens_t, workspace_starts_t + return cache, dst, block_table, workspace_starts_t def bench_scenario(label, num_reqs, total_tokens_list, save_path): @@ -94,7 +93,7 @@ def bench_scenario(label, num_reqs, total_tokens_list, save_path): ) ) def bench_fn(total_tokens, provider, num_reqs): - cache, dst, block_table, seq_lens_t, ws_starts = make_inputs( + cache, dst, block_table, ws_starts = make_inputs( total_tokens, num_reqs, BLOCK_SIZE ) @@ -102,7 +101,7 @@ def bench_scenario(label, num_reqs, total_tokens_list, save_path): ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( lambda: ops.cp_gather_and_upconvert_fp8_kv_cache( - cache, dst, block_table, seq_lens_t, ws_starts, num_reqs + cache, dst, block_table, ws_starts, num_reqs ), quantiles=quantiles, rep=500, diff --git a/csrc/cache.h b/csrc/cache.h index a9e74b0dc2..dbc2675fc7 100644 --- a/csrc/cache.h +++ b/csrc/cache.h @@ -67,9 +67,8 @@ void cp_gather_and_upconvert_fp8_kv_cache( torch::Tensor const& src_cache, // [NUM_BLOCKS, BLOCK_SIZE, 656] torch::Tensor const& dst, // [TOT_TOKENS, 576] torch::Tensor const& block_table, // [BATCH, BLOCK_INDICES] - torch::Tensor const& seq_lens, // [BATCH] torch::Tensor const& workspace_starts, // [BATCH] - int64_t batch_size); + int64_t batch_size, std::optional seq_starts = std::nullopt); // Indexer K quantization and cache function void indexer_k_quant_and_cache( diff --git a/csrc/libtorch_stable/cache_kernels.cu b/csrc/libtorch_stable/cache_kernels.cu index a1ac81cb10..2d4b47b4b4 100644 --- a/csrc/libtorch_stable/cache_kernels.cu +++ b/csrc/libtorch_stable/cache_kernels.cu @@ -1174,7 +1174,8 @@ __global__ void cp_gather_and_upconvert_fp8_kv_cache( const int32_t num_reqs, const int32_t block_size, const int32_t total_tokens, const int64_t block_table_stride, const int64_t cache_block_stride, const int64_t cache_entry_stride, - const int64_t dst_entry_stride) { + const int64_t dst_entry_stride, + const int32_t* __restrict__ seq_starts) { // Optional source offsets const int flat_warp_id = (blockIdx.x * blockDim.x + threadIdx.x) >> 5; if (flat_warp_id >= total_tokens) return; const int lane_id = threadIdx.x & 31; @@ -1192,7 +1193,8 @@ __global__ void cp_gather_and_upconvert_fp8_kv_cache( // Compute physical token address via block table const int out_token_id = flat_warp_id; - const int token_offset = out_token_id - workspace_starts[req_id]; + int token_offset = out_token_id - workspace_starts[req_id]; + if (seq_starts != nullptr) token_offset += seq_starts[req_id]; const int cache_block_idx = token_offset / block_size; const int offset_in_block = token_offset % block_size; const int physical_block = @@ -1383,9 +1385,9 @@ void cp_gather_and_upconvert_fp8_kv_cache( torch::stable::Tensor const& src_cache, // [NUM_BLOCKS, BLOCK_SIZE, 656] torch::stable::Tensor const& dst, // [TOT_TOKENS, 576] torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES] - torch::stable::Tensor const& seq_lens, // [BATCH] torch::stable::Tensor const& workspace_starts, // [BATCH] - int64_t batch_size) { + int64_t batch_size, + std::optional seq_starts = std::nullopt) { torch::stable::accelerator::DeviceGuard device_guard( src_cache.get_device_index()); const cudaStream_t stream = get_current_cuda_stream(); @@ -1396,20 +1398,25 @@ void cp_gather_and_upconvert_fp8_kv_cache( STD_TORCH_CHECK( block_table.scalar_type() == torch::headeronly::ScalarType::Int, "block_table must be int32"); - STD_TORCH_CHECK(seq_lens.scalar_type() == torch::headeronly::ScalarType::Int, - "seq_lens must be int32"); STD_TORCH_CHECK( workspace_starts.scalar_type() == torch::headeronly::ScalarType::Int, "workspace_starts must be int32"); + if (seq_starts.has_value()) { + STD_TORCH_CHECK( + seq_starts.value().scalar_type() == torch::headeronly::ScalarType::Int, + "seq_starts must be int32"); + } STD_TORCH_CHECK(src_cache.device() == dst.device(), "src_cache and dst must be on the same device"); STD_TORCH_CHECK(src_cache.device() == block_table.device(), "src_cache and block_table must be on the same device"); - STD_TORCH_CHECK(src_cache.device() == seq_lens.device(), - "src_cache and seq_lens must be on the same device"); STD_TORCH_CHECK(src_cache.device() == workspace_starts.device(), "src_cache and workspace_starts must be on the same device"); + if (seq_starts.has_value()) { + STD_TORCH_CHECK(src_cache.device() == seq_starts.value().device(), + "src_cache and seq_starts must be on the same device"); + } auto dtype = src_cache.scalar_type(); STD_TORCH_CHECK( dtype == torch::headeronly::ScalarType::Byte || // uint8 @@ -1438,6 +1445,9 @@ void cp_gather_and_upconvert_fp8_kv_cache( constexpr int warps_per_block = 8; const int grid_size = (total_tokens + warps_per_block - 1) / warps_per_block; const int block_size_threads = warps_per_block * 32; // 256 threads + const int32_t* seq_starts_ptr = + seq_starts.has_value() ? seq_starts.value().const_data_ptr() + : nullptr; vllm::cp_gather_and_upconvert_fp8_kv_cache<<>>( @@ -1446,7 +1456,7 @@ void cp_gather_and_upconvert_fp8_kv_cache( workspace_starts.const_data_ptr(), static_cast(batch_size), block_size, total_tokens, block_table_stride, cache_block_stride, cache_entry_stride, - dst_entry_stride); + dst_entry_stride, seq_starts_ptr); } // Macro to dispatch the kernel based on the data type. diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index ae274226fe..3834bea585 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -527,9 +527,9 @@ void cp_gather_and_upconvert_fp8_kv_cache( // 656] torch::stable::Tensor const& dst, // [TOT_TOKENS, 576] torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES] - torch::stable::Tensor const& seq_lens, // [BATCH] torch::stable::Tensor const& workspace_starts, // [BATCH] - int64_t batch_size); + int64_t batch_size, + std::optional seq_starts = std::nullopt); // Indexer K quantization and cache function void indexer_k_quant_and_cache( diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index fd2ffe92e0..0a475d02c6 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -847,8 +847,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) { ops.def( "cp_gather_and_upconvert_fp8_kv_cache(Tensor src_cache, Tensor! dst, " - "Tensor block_table, Tensor seq_lens, Tensor workspace_starts, int " - "batch_size) -> ()"); + "Tensor block_table, Tensor workspace_starts, int batch_size, Tensor? " + "seq_starts) -> ()"); ops.def( "indexer_k_quant_and_cache(Tensor k, Tensor! kv_cache, Tensor " diff --git a/tests/kernels/test_cp_gather_fp8.py b/tests/kernels/test_cp_gather_fp8.py index d9ee8defdb..36e7f0c9d4 100644 --- a/tests/kernels/test_cp_gather_fp8.py +++ b/tests/kernels/test_cp_gather_fp8.py @@ -27,8 +27,8 @@ def _build_test_case(seq_lens, block_size, seed=42): seed: Random seed for reproducibility. Returns: - Tuple of (cache, block_table, seq_lens_t, workspace_starts_t, - num_reqs, total_tokens, expected_output). + Tuple of (cache, block_table, workspace_starts_t, num_reqs, + total_tokens, expected_output). """ torch.manual_seed(seed) @@ -112,7 +112,6 @@ def _build_test_case(seq_lens, block_size, seed=42): # Expected output: exact copy expected[out_idx, NOPE_DIM:] = rope - seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda") workspace_starts_t = torch.tensor( workspace_starts, dtype=torch.int32, device="cuda" ) @@ -120,7 +119,6 @@ def _build_test_case(seq_lens, block_size, seed=42): return ( cache, block_table, - seq_lens_t, workspace_starts_t, num_reqs, total_tokens, @@ -197,7 +195,6 @@ def _build_test_case_fast(seq_lens, block_size, seed=42): flat_cache[:sl] = token_data[ws : ws + sl] block_start += nb - seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda") workspace_starts_t = torch.tensor( workspace_starts, dtype=torch.int32, device="cuda" ) @@ -205,7 +202,6 @@ def _build_test_case_fast(seq_lens, block_size, seed=42): return ( cache, block_table, - seq_lens_t, workspace_starts_t, num_reqs, total_tokens, @@ -232,7 +228,6 @@ def test_cp_gather_and_upconvert_fp8_kv_cache(seq_lens, block_size): ( cache, block_table, - seq_lens_t, workspace_starts_t, num_reqs, total_tokens, @@ -244,7 +239,7 @@ def test_cp_gather_and_upconvert_fp8_kv_cache(seq_lens, block_size): ) ops.cp_gather_and_upconvert_fp8_kv_cache( - cache, dst, block_table, seq_lens_t, workspace_starts_t, num_reqs + cache, dst, block_table, workspace_starts_t, num_reqs ) # NoPE: fp8 dequant has rounding error, so we allow small tolerance. @@ -277,8 +272,6 @@ def test_cp_gather_fp8_shuffled_blocks(): ) block_table = torch.tensor([[3, 1]], dtype=torch.int32, device="cuda") workspace_starts = torch.tensor([0], dtype=torch.int32, device="cuda") - seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda") - expected = torch.zeros( total_tokens, NOPE_DIM + ROPE_DIM, dtype=torch.bfloat16, device="cuda" ) @@ -314,7 +307,7 @@ def test_cp_gather_fp8_shuffled_blocks(): ) ops.cp_gather_and_upconvert_fp8_kv_cache( - cache, dst, block_table, seq_lens_t, workspace_starts, len(seq_lens) + cache, dst, block_table, workspace_starts, len(seq_lens) ) torch.testing.assert_close( @@ -323,6 +316,64 @@ def test_cp_gather_fp8_shuffled_blocks(): assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:]) +@pytest.mark.parametrize( + "gather_seq_lens,seq_starts", + [ + ([6, 5, 3], [3, 4, 5]), + ([0, 5, 3], [12, 4, 5]), + ], +) +def test_cp_gather_fp8_with_sequence_starts(gather_seq_lens, seq_starts): + """Gather request slices beginning at arbitrary cache positions.""" + full_seq_lens = [12, 11, 9] + ( + cache, + block_table, + _workspace_starts_t, + num_reqs, + _total_tokens, + full_expected, + ) = _build_test_case(full_seq_lens, block_size=4) + + workspace_starts = torch.tensor( + [0, gather_seq_lens[0], sum(gather_seq_lens[:2])], + dtype=torch.int32, + device="cuda", + ) + seq_starts_t = torch.tensor(seq_starts, dtype=torch.int32, device="cuda") + dst = torch.empty( + sum(gather_seq_lens), + NOPE_DIM + ROPE_DIM, + dtype=torch.bfloat16, + device="cuda", + ) + + ops.cp_gather_and_upconvert_fp8_kv_cache( + cache, + dst, + block_table, + workspace_starts, + num_reqs, + seq_starts_t, + ) + + full_workspace_starts = [0, full_seq_lens[0], sum(full_seq_lens[:2])] + expected = torch.cat( + [ + full_expected[ + full_workspace_starts[i] + seq_starts[i] : full_workspace_starts[i] + + seq_starts[i] + + gather_seq_lens[i] + ] + for i in range(num_reqs) + ] + ) + torch.testing.assert_close( + dst[:, :NOPE_DIM], expected[:, :NOPE_DIM], atol=1e-3, rtol=1e-2 + ) + assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:]) + + @pytest.mark.parametrize( "seq_lens,block_size", [ @@ -342,7 +393,6 @@ def test_cp_gather_fp8_large_seqlens(seq_lens, block_size): ( cache, block_table, - seq_lens_t, workspace_starts_t, num_reqs, total_tokens, @@ -354,7 +404,7 @@ def test_cp_gather_fp8_large_seqlens(seq_lens, block_size): ) ops.cp_gather_and_upconvert_fp8_kv_cache( - cache, dst, block_table, seq_lens_t, workspace_starts_t, num_reqs + cache, dst, block_table, workspace_starts_t, num_reqs ) torch.testing.assert_close( diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index 3afa8eca92..b8dd1f4c51 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -42,7 +42,10 @@ from vllm.v1.attention.backends.mla.prefill import ( ) from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported -from vllm.v1.kv_cache_interface import MLAAttentionSpec +from vllm.v1.kv_cache_interface import ( + KVQuantMode, + MLAAttentionSpec, +) BACKENDS_TO_TEST = [ AttentionBackendEnum.CUTLASS_MLA, @@ -55,6 +58,31 @@ BACKENDS_TO_TEST = [ DEVICE_TYPE = current_platform.device_type + +@pytest.mark.parametrize( + ("cache_dtype", "expected_quant_mode"), + [ + ("auto", KVQuantMode.NONE), + ("fp8_ds_mla", KVQuantMode.FP8_PER_TENSOR), + ], +) +def test_mla_kv_cache_spec_uses_layer_cache_dtype( + cache_dtype: str, expected_quant_mode: KVQuantMode +): + layer = SimpleNamespace(kv_cache_dtype=cache_dtype, head_size=576) + vllm_config = SimpleNamespace( + cache_config=SimpleNamespace(block_size=64), model_config=None + ) + + spec = MLAAttention.get_kv_cache_spec(layer, vllm_config) + + assert isinstance(spec, MLAAttentionSpec) + assert spec.cache_dtype_str == cache_dtype + assert spec.kv_quant_mode == expected_quant_mode + if cache_dtype == "fp8_ds_mla": + assert spec.page_size_bytes == 64 * 656 + + # Remove sm100 backends from the list if not using sm100 if not torch.cuda.is_available() or torch.cuda.get_device_properties(0).major < 10: BACKENDS_TO_TEST.remove(AttentionBackendEnum.CUTLASS_MLA) diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index d88f107ece..6ec9fc25ed 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -40,10 +40,16 @@ from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import ( ) from vllm.v1.attention.backends.mla.flashmla_sparse import ( FlashMLASparseBackend, + FlashMLASparseImpl, + FlashMLASparseMetadata, + FlashMLASparseMetadataBuilder, triton_convert_req_index_to_global_index, ) from vllm.v1.attention.backends.mla.indexer import split_indexer_prefill_chunks -from vllm.v1.attention.backends.utils import split_prefill_chunks +from vllm.v1.attention.backends.utils import ( + split_decodes_and_prefills, + split_prefill_chunks, +) from vllm.v1.attention.ops import flashmla SPARSE_BACKEND_BATCH_SPECS = { @@ -721,6 +727,120 @@ def test_triton_convert_req_index_to_global_index_with_prefill_workspace(block_s torch.testing.assert_close(result, reference_result, rtol=0, atol=0) +@pytest.mark.skipif( + torch.cuda.get_device_capability() < (9, 0), + reason="FlashMLASparseBackend requires CUDA 9.0 or higher", +) +def test_triton_convert_rejects_req_id_longer_than_token_indices(): + """Guard against the #47327 regression: the kernel grid is sized by + req_id but the output is allocated like token_indices, so a full-batch + req_id combined with an MQA-subset token_indices wrote past the end of + the output buffer. The wrapper must reject the length mismatch instead + of corrupting memory.""" + device = torch.device(DEVICE_TYPE) + num_topk_tokens = 128 + block_size = 64 + block_table = torch.arange(40, dtype=torch.int32, device=device).view(4, 10) + + # Full batch: 2 decode tokens + 10 prefill tokens + req_id_full = torch.tensor( + [0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3], dtype=torch.int32, device=device + ) + num_mqa_tokens = 2 + token_indices = torch.randint( + 0, + block_size * 10, + (num_mqa_tokens, num_topk_tokens), + dtype=torch.int32, + device=device, + ) + + with pytest.raises(AssertionError, match="must cover the same tokens"): + triton_convert_req_index_to_global_index( + req_id_full, + block_table, + token_indices, + BLOCK_SIZE=block_size, + NUM_TOPK_TOKENS=num_topk_tokens, + ) + + # The sliced call is the intended usage and must match the reference. + result = triton_convert_req_index_to_global_index( + req_id_full[:num_mqa_tokens], + block_table, + token_indices, + BLOCK_SIZE=block_size, + NUM_TOPK_TOKENS=num_topk_tokens, + ) + reference = _triton_convert_reference_impl( + req_id_full[:num_mqa_tokens], + block_table, + token_indices, + block_size, + num_topk_tokens, + ) + torch.testing.assert_close(result, reference, rtol=0, atol=0) + + +@pytest.mark.skipif( + torch.cuda.get_device_capability() < (9, 0), + reason="FlashMLASparseBackend requires CUDA 9.0 or higher", +) +def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens(): + """Guard against the #47327 regression: when the dense-MHA prefill split + is active, forward_mqa only receives the leading decode tokens, but + _forward_bf16_kv passed the full-batch req_id_per_token to the index + conversion, making it write past the end of its output buffer. The call + site must slice req_id_per_token to the MQA tokens.""" + device = torch.device(DEVICE_TYPE) + num_topk_tokens = 128 + block_size = 64 + num_batch_tokens = 12 + num_mqa_tokens = 2 + + attn_metadata = SimpleNamespace( + req_id_per_token=torch.tensor( + [0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32, device=device + ), + block_table=torch.arange(40, dtype=torch.int32, device=device).view(4, 10), + block_size=block_size, + ) + assert attn_metadata.req_id_per_token.shape[0] == num_batch_tokens + + q = torch.zeros(num_mqa_tokens, 4, 576, dtype=torch.bfloat16, device=device) + kv_cache = torch.zeros(40 * block_size, 576, dtype=torch.bfloat16, device=device) + topk_indices = torch.randint( + 0, + block_size * 10, + (num_mqa_tokens, num_topk_tokens), + dtype=torch.int32, + device=device, + ) + + captured = {} + + def _stub_kernel(q, kv, indices, lengths): + captured["indices"] = indices + return torch.zeros(q.shape[0], q.shape[1], 512, dtype=q.dtype, device=q.device) + + stub_impl = SimpleNamespace(_bf16_flash_mla_kernel=_stub_kernel) + + out = FlashMLASparseImpl._forward_bf16_kv( + stub_impl, q, kv_cache, topk_indices, attn_metadata + ) + + assert out.shape[0] == num_mqa_tokens + assert captured["indices"].shape[0] == num_mqa_tokens + reference = _triton_convert_reference_impl( + attn_metadata.req_id_per_token[:num_mqa_tokens], + attn_metadata.block_table, + topk_indices, + block_size, + num_topk_tokens, + ) + torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0) + + @pytest.mark.parametrize( "seq_lens,max_buf,expected", [ @@ -1124,3 +1244,203 @@ def test_triton_convert_returns_valid_counts(): ) assert isinstance(result_only, torch.Tensor) torch.testing.assert_close(result_only, result, rtol=0, atol=0) + + +def test_flashmla_cache_dtype_aliases_use_ds_layout(): + from vllm.model_executor.layers.attention.mla_attention import ( + _canonicalize_sparse_mla_kv_cache_dtype, + ) + + # kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer + # stores kv_cache_dtype, so they cannot bypass the gate. + for alias in ("fp8", "fp8_e4m3"): + assert ( + _canonicalize_sparse_mla_kv_cache_dtype(FlashMLASparseBackend, alias) + == "fp8_ds_mla" + ) + + +def test_flashmla_fp8_metadata_reuses_common_batch_split(): + builder = SimpleNamespace( + device=torch.device(DEVICE_TYPE), + vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=8)), + ) + common_metadata = SimpleNamespace( + num_actual_tokens=1, + seq_lens_cpu_upper_bound=torch.tensor([1]), + query_start_loc_cpu=torch.tensor([0, 1]), + block_table_tensor=torch.zeros(1, 1, dtype=torch.int32, device=DEVICE_TYPE), + ) + metadata = FlashMLASparseMetadata( + num_reqs=1, + max_query_len=1, + max_seq_len=1, + num_actual_tokens=1, + query_start_loc=torch.tensor([0, 1], device=DEVICE_TYPE), + slot_mapping=torch.tensor([0], device=DEVICE_TYPE), + block_table=torch.zeros(1, 1, dtype=torch.int32, device=DEVICE_TYPE), + req_id_per_token=torch.zeros(1, dtype=torch.int32, device=DEVICE_TYPE), + num_decodes=0, + num_prefills=1, + num_decode_tokens=0, + ) + + fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode( + builder, common_metadata, metadata + ) + + assert fp8_metadata.num_decodes == 0 + assert fp8_metadata.num_prefills == 1 + assert fp8_metadata.num_decode_tokens == 0 + assert fp8_metadata.num_prefill_tokens == 1 + + +def test_flashmla_common_metadata_requires_uniform_decodes(): + common_metadata = SimpleNamespace( + max_query_len=3, + num_reqs=3, + num_actual_tokens=6, + query_start_loc_cpu=torch.tensor([0, 1, 3, 6]), + is_prefilling=None, + ) + + split = split_decodes_and_prefills( + common_metadata, + decode_threshold=128, + require_uniform=FlashMLASparseMetadataBuilder.require_uniform_decodes, + ) + + assert split == (1, 2, 1, 5) + + +def test_flashmla_fp8_metadata_excludes_zero_token_decode_padding(monkeypatch): + monkeypatch.setattr( + "vllm.v1.attention.backends.mla.flashmla_sparse.get_mla_metadata", + lambda: (object(), None), + ) + builder = SimpleNamespace( + device=torch.device(DEVICE_TYPE), + dummy_block_table=torch.zeros(7, 1, device=DEVICE_TYPE), + max_model_len_tensor=torch.zeros(7, device=DEVICE_TYPE), + ) + query_start_loc_cpu = torch.tensor([0, 110, 220, 330, 440, 550, 660, 660]) + common_metadata = SimpleNamespace( + num_actual_tokens=660, + query_start_loc_cpu=query_start_loc_cpu, + seq_lens=torch.arange(7, device=DEVICE_TYPE), + ) + metadata = FlashMLASparseMetadata( + num_reqs=7, + max_query_len=110, + max_seq_len=110, + num_actual_tokens=660, + query_start_loc=query_start_loc_cpu.to(DEVICE_TYPE), + slot_mapping=torch.arange(660, device=DEVICE_TYPE), + block_table=torch.zeros(7, 1, dtype=torch.int32, device=DEVICE_TYPE), + req_id_per_token=torch.zeros(660, dtype=torch.int32, device=DEVICE_TYPE), + num_decodes=7, + num_prefills=0, + num_decode_tokens=660, + ) + + fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode( + builder, common_metadata, metadata + ) + + assert fp8_metadata.num_decodes == 6 + assert fp8_metadata.num_decode_tokens == 660 + assert fp8_metadata.decode is not None + assert fp8_metadata.decode.decode_query_len == 110 + torch.testing.assert_close( + fp8_metadata.decode.seq_lens, torch.arange(6, device=DEVICE_TYPE) + ) + + +@pytest.mark.parametrize("use_mixed_batch", [False, True]) +def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: bool): + num_decode_tokens = 2 + num_batch_tokens = 5 + q = torch.empty(num_decode_tokens, 2, 3, device=DEVICE_TYPE) + topk_indices = torch.empty(num_decode_tokens, 4, device=DEVICE_TYPE) + kernel_q_shapes = [] + + def convert_indices(*args, **kwargs): # noqa: ARG001 + assert not kwargs.get("HAS_PREFILL_WORKSPACE", False) + if not kwargs.get("return_valid_counts", False): + return topk_indices + valid_counts = torch.full( + (num_decode_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE + ) + return topk_indices, valid_counts + + monkeypatch.setattr( + "vllm.v1.attention.backends.mla.flashmla_sparse." + "triton_convert_req_index_to_global_index", + convert_indices, + ) + + def run_kernel(**kwargs): + kernel_q_shapes.append(kwargs["q"].shape) + return kwargs["q"][..., :1], None + + if use_mixed_batch: + fp8_metadata = FlashMLASparseMetadata.FP8KernelMetadata( + scheduler_metadata=object(), # type: ignore[arg-type] + dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE), + cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE), + ) + else: + FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode + fp8_metadata = FP8Meta( + num_decodes=1, + num_prefills=1, + num_decode_tokens=num_decode_tokens, + num_prefill_tokens=num_batch_tokens - num_decode_tokens, + decode=FP8Meta.Decode( + seq_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE), + kernel_metadata=object(), # type: ignore[arg-type] + decode_query_len=num_decode_tokens, + ), + prefill=FP8Meta.Prefill( + request_ids=torch.empty( + num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE + ), + workspace_starts=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE), + chunks=[], + ), + ) + metadata = SimpleNamespace( + fp8_extra_metadata=fp8_metadata, + fp8_use_mixed_batch=use_mixed_batch, + num_actual_tokens=num_batch_tokens, + req_id_per_token=torch.empty( + num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE + ), + block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE), + block_size=64, + ) + impl = SimpleNamespace( + kv_cache_dtype="fp8_ds_mla", + topk_indices_buffer=topk_indices, + num_heads=2, + kv_lora_rank=1, + _fp8_flash_mla_kernel=run_kernel, + ) + impl._forward_fp8_kv_mixed_batch = MethodType( + FlashMLASparseImpl._forward_fp8_kv_mixed_batch, impl + ) + impl._forward_fp8_kv_separate_prefill_decode = MethodType( + FlashMLASparseImpl._forward_fp8_kv_separate_prefill_decode, impl + ) + + output, lse = FlashMLASparseImpl.forward_mqa( + impl, + q, + torch.empty(0, device=DEVICE_TYPE), + metadata, + None, + ) + + assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)] + assert output.shape == (num_decode_tokens, 2, 1) + assert lse is None diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 9fcc4ae87e..599cac0ed6 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2759,9 +2759,9 @@ def cp_gather_and_upconvert_fp8_kv_cache( src_cache: torch.Tensor, dst: torch.Tensor, block_table: torch.Tensor, - seq_lens: torch.Tensor, workspace_starts: torch.Tensor, batch_size: int, + seq_starts: torch.Tensor | None = None, ) -> None: """Gather and upconvert FP8 KV cache to BF16 workspace. @@ -2769,12 +2769,12 @@ def cp_gather_and_upconvert_fp8_kv_cache( src_cache: FP8 KV cache [num_blocks, block_size, 656] dst: BF16 output workspace [total_tokens, 576] block_table: Block indices [num_reqs, max_blocks] - seq_lens: Sequence lengths [num_reqs] workspace_starts: Workspace start offsets [num_reqs] batch_size: Number of requests + seq_starts: Optional source sequence offsets [num_reqs] """ torch.ops._C_cache_ops.cp_gather_and_upconvert_fp8_kv_cache( - src_cache, dst, block_table, seq_lens, workspace_starts, batch_size + src_cache, dst, block_table, workspace_starts, batch_size, seq_starts ) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 021f0b02b7..5bb0b43c66 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -1749,6 +1749,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): self.determine_chunked_prefill_workspace_size(vllm_config) ) + use_packed_fp8_cache = vllm_config.cache_config.cache_dtype == "fp8_ds_mla" if self.dcp_world_size > 1: # Note(hc): The local kvcache is incomplete when DCP is triggered, # an additional kvcache allgather across the DCP group is therefore @@ -1761,7 +1762,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): + self.chunked_prefill_workspace_size // self.dcp_world_size, self.model_config.get_head_size(), ), - dtype=self.model_config.dtype, + dtype=torch.bfloat16 + if use_packed_fp8_cache + else self.model_config.dtype, device=device, ) else: @@ -1770,7 +1773,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): self.chunked_prefill_workspace_size, self.model_config.get_head_size(), ), - dtype=self.q_data_type, + dtype=torch.bfloat16 if use_packed_fp8_cache else self.q_data_type, device=device, ) @@ -2115,7 +2118,16 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]): for i in range(iters): toks = prefill_metadata.chunked_context.seq_tot[i] - if not use_fp8_prefill: + if self.kv_cache_dtype == "fp8_ds_mla": + ops.cp_gather_and_upconvert_fp8_kv_cache( + src_cache=kv_c_and_k_pe_cache, + dst=workspace[:toks], + block_table=prefill_metadata.block_table, + workspace_starts=prefill_metadata.chunked_context.cu_seq_lens[i], + batch_size=attn_metadata.num_prefills, + seq_starts=prefill_metadata.chunked_context.starts[i], + ) + elif not use_fp8_prefill: ops.gather_and_maybe_dequant_cache( src_cache=kv_c_and_k_pe_cache, dst=workspace, @@ -2230,9 +2242,16 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]): padded_local_cu_seq_lens = ( prefill_metadata.chunked_context.padded_local_cu_seq_lens[i] ) - if is_quantized_kv_cache(self.kv_cache_dtype) and ( - self.kv_cache_dtype != "fp8_ds_mla" - ): + if self.kv_cache_dtype == "fp8_ds_mla": + ops.cp_gather_and_upconvert_fp8_kv_cache( + src_cache=kv_c_and_k_pe_cache, + dst=workspace[:toks], + block_table=prefill_metadata.block_table, + workspace_starts=padded_local_cu_seq_lens, + batch_size=attn_metadata.num_prefills, + seq_starts=prefill_metadata.chunked_context.starts[i], + ) + elif is_quantized_kv_cache(self.kv_cache_dtype): assert k_scale is not None ops.gather_and_maybe_dequant_cache( src_cache=kv_c_and_k_pe_cache, diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index 4948dca676..35bbd4d3d4 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -3,7 +3,7 @@ """Shared forward_mha implementation and metadata builder for sparse MLA backends.""" from shutil import which -from typing import TYPE_CHECKING, Generic, TypeVar +from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar import numpy as np import torch @@ -38,6 +38,7 @@ T = TypeVar("T", bound=AttentionMetadata) class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): metadata_cls: type[T] + require_uniform_decodes: ClassVar[bool] = False def __init__( self, @@ -173,6 +174,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): num_decodes, num_prefills, num_decode_tokens, _ = split_decodes_and_prefills( common_attn_metadata, decode_threshold=self.reorder_batch_threshold or 1, + require_uniform=self.require_uniform_decodes, ) ( prefill_query_start_loc, diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 4dea36a3de..e8be7ad5bb 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -32,7 +32,6 @@ from vllm.v1.attention.backends.mla.sparse_utils import ( from vllm.v1.attention.backends.utils import ( reshape_attn_output_for_spec_decode, reshape_query_for_spec_decode, - split_decodes_and_prefills, split_prefill_chunks, ) from vllm.v1.attention.ops.flashmla import ( @@ -57,8 +56,8 @@ logger = init_logger(__name__) # the FP8 decode kernel for decode. # Currently we use #1 when the number of heads per rank is low (i.e. TP) since the BF16 # prefill kernel requires padding the number of heads to 128 while the decode does not -# so when the per ranke head count is below MIN_HEADS_FOR_BF16_PREFILL we use the mixed -# batch mode (#2). +# so when the per-rank head count is below MIN_HEADS_FOR_BF16_PREFILL we use the mixed +# batch mode (#1). MIN_HEADS_FOR_BF16_PREFILL = 32 """ @@ -182,10 +181,6 @@ class FlashMLASparseMetadata(AttentionMetadata): @dataclass class Prefill: - # Sequence lengths (context + query) for prefill requests - # Shape: [num_prefill_reqs] - seq_lens: torch.Tensor - # Request ID for each token: -1 for decode tokens, request index # (0, 1, 2, ...) for prefill tokens. # Shape: [num_actual_tokens] @@ -204,7 +199,6 @@ class FlashMLASparseMetadata(AttentionMetadata): Prefill requests may be chunked to fit within the fixed workspace size. """ - seq_lens: torch.Tensor tokens_slice: slice block_table: torch.Tensor req_start_idx: int @@ -239,6 +233,7 @@ class FlashMLASparseMetadataBuilder( SparseMLACommonMetadataBuilder[FlashMLASparseMetadata] ): _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + require_uniform_decodes: ClassVar[bool] = True metadata_cls = FlashMLASparseMetadata def __init__( @@ -322,10 +317,11 @@ class FlashMLASparseMetadataBuilder( self, common_attn_metadata: CommonAttentionMetadata, ) -> "FlashMLASparseMetadata.FP8KernelMetadata": - """Build FP8 metadata treating all tokens as one mixed batch. + """Build FP8 metadata treating MQA tokens as one batch. - This matches main branch's approach and avoids the BF16 prefill kernel - which has head padding overhead when num_heads is small (high TP case). + The scheduler initializes lazily from the runtime query shape, which may + be the full batch or only decodes when prefills use dense MHA. This avoids + the BF16 prefill kernel's head-padding overhead at high TP. """ num_tokens = common_attn_metadata.num_actual_tokens @@ -353,20 +349,29 @@ class FlashMLASparseMetadataBuilder( def _build_fp8_separate_prefill_decode( self, common_attn_metadata: CommonAttentionMetadata, + metadata: FlashMLASparseMetadata, ) -> "FlashMLASparseMetadata.FP8SeparatePrefillDecode": num_tokens = common_attn_metadata.num_actual_tokens (num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens) = ( - split_decodes_and_prefills( - common_attn_metadata, - decode_threshold=self.reorder_batch_threshold or 1, - require_uniform=True, - ) + metadata.num_decodes, + metadata.num_prefills, + metadata.num_decode_tokens, + num_tokens - metadata.num_decode_tokens, ) + decode_query_len = 0 + active_num_decodes = num_decodes + if num_decodes > 0: + query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu + decode_query_len = (query_start_loc_cpu[1] - query_start_loc_cpu[0]).item() + assert decode_query_len > 0 + active_num_decodes = num_decode_tokens // decode_query_len + assert active_num_decodes * decode_query_len == num_decode_tokens + FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode fp8_metadata = FP8Meta( - num_decodes=num_decodes, + num_decodes=active_num_decodes, num_prefills=num_prefills, num_decode_tokens=num_decode_tokens, num_prefill_tokens=num_prefill_tokens, @@ -374,7 +379,6 @@ class FlashMLASparseMetadataBuilder( # Extract prefill sequence lengths (context + query, not just query) # Decode requests come first in the batch, prefill requests follow - prefill_seq_lens = None prefill_request_id = None prefill_workspace_starts = None prefill_chunks = None @@ -386,11 +390,9 @@ class FlashMLASparseMetadataBuilder( # slice below), so no D2H sync is needed. seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound assert seq_lens_cpu is not None - seq_lens = common_attn_metadata.seq_lens query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu prefill_seq_lens_cpu = seq_lens_cpu[num_decodes:] - prefill_seq_lens = seq_lens[num_decodes:] # Build prefill_request_id: -1 for decode, request index for # prefill. This enables a single @@ -438,7 +440,6 @@ class FlashMLASparseMetadataBuilder( offset = prefill_workspace_starts_cpu[chunk_start].item() prefill_workspace_starts_cpu[chunk_start:chunk_end] -= offset - chunk_seq_lens = prefill_seq_lens[chunk_start:chunk_end] chunk_tot_seqlen = prefill_seq_lens_cpu[chunk_start:chunk_end].sum() token_start = query_start_loc_cpu[num_decodes + chunk_start].item() token_end = query_start_loc_cpu[num_decodes + chunk_end].item() @@ -452,7 +453,6 @@ class FlashMLASparseMetadataBuilder( prefill_chunks.append( FP8Meta.Prefill.Chunk( - seq_lens=chunk_seq_lens, tokens_slice=tokens_slice, block_table=chunk_block_table, req_start_idx=chunk_start, @@ -466,27 +466,22 @@ class FlashMLASparseMetadataBuilder( ) fp8_metadata.prefill = FP8Meta.Prefill( - seq_lens=prefill_seq_lens, request_ids=prefill_request_id, workspace_starts=prefill_workspace_starts, chunks=prefill_chunks, ) if num_decodes > 0: - # Compute decode_query_len for spec decode (uniform due to require_uniform) - query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu - decode_query_len = (query_start_loc_cpu[1] - query_start_loc_cpu[0]).item() - # Use padded head count since that's what the kernel will see scheduler_metadata, _ = get_mla_metadata() kernel_meta = FlashMLASparseMetadata.FP8KernelMetadata( scheduler_metadata=scheduler_metadata, - dummy_block_table=self.dummy_block_table[:num_decodes], - cache_lens=self.max_model_len_tensor[:num_decodes], + dummy_block_table=self.dummy_block_table[:active_num_decodes], + cache_lens=self.max_model_len_tensor[:active_num_decodes], ) fp8_metadata.decode = FP8Meta.Decode( - seq_lens=common_attn_metadata.seq_lens[:num_decodes], + seq_lens=common_attn_metadata.seq_lens[:active_num_decodes], kernel_metadata=kernel_meta, decode_query_len=decode_query_len, ) @@ -510,7 +505,7 @@ class FlashMLASparseMetadataBuilder( ) else: metadata.fp8_extra_metadata = self._build_fp8_separate_prefill_decode( - common_attn_metadata + common_attn_metadata, metadata ) return metadata @@ -597,9 +592,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): attn_metadata: FlashMLASparseMetadata, ) -> torch.Tensor: # Convert per-request indices to global slots (decode) or workspace - # offsets (prefill). + # offsets (prefill). req_id_per_token covers the whole batch; slice it + # to the MQA tokens (q may exclude prefill tokens routed to dense MHA). topk_indices, topk_length = triton_convert_req_index_to_global_index( - attn_metadata.req_id_per_token, + attn_metadata.req_id_per_token[: topk_indices.shape[0]], attn_metadata.block_table, topk_indices, BLOCK_SIZE=attn_metadata.block_size, @@ -624,11 +620,18 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): fp8_metadata = attn_metadata.fp8_extra_metadata assert isinstance(fp8_metadata, FlashMLASparseMetadata.FP8SeparatePrefillDecode) num_decodes = fp8_metadata.num_decodes + num_mqa_tokens = q.shape[0] + num_decode_tokens = fp8_metadata.num_decode_tokens + num_prefill_tokens = num_mqa_tokens - num_decode_tokens + assert num_prefill_tokens in (0, fp8_metadata.num_prefill_tokens), ( + "FP8 sparse MLA expects either the decode subset or the full batch" + ) prefill_request_ids = None prefill_workspace_starts = None has_prefill_workspace = False - if fp8_metadata.prefill is not None: + if num_prefill_tokens > 0: + assert fp8_metadata.prefill is not None prefill_request_ids = fp8_metadata.prefill.request_ids prefill_workspace_starts = fp8_metadata.prefill.workspace_starts has_prefill_workspace = True @@ -640,7 +643,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): # prefill_workspace_starts has been adjusted in-place per chunk so # prefill indices automatically come out chunk-local topk_indices, topk_length = triton_convert_req_index_to_global_index( - attn_metadata.req_id_per_token, + attn_metadata.req_id_per_token[: topk_indices.shape[0]], attn_metadata.block_table, topk_indices, BLOCK_SIZE=attn_metadata.block_size, @@ -676,9 +679,6 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): # -> (num_decode_tokens, num_heads, head_dim_v) return reshape_attn_output_for_spec_decode(attn_out) - num_decode_tokens = fp8_metadata.num_decode_tokens - num_prefill_tokens = fp8_metadata.num_prefill_tokens - # Pure decode: direct call without allocation if num_decode_tokens > 0 and num_prefill_tokens == 0: assert fp8_metadata.decode is not None @@ -686,7 +686,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): else: # Mixed or pure prefill: allocate output tensor attn_out = q.new_empty( - (attn_metadata.num_actual_tokens, self.num_heads, self.kv_lora_rank), + (num_mqa_tokens, self.num_heads, self.kv_lora_rank), dtype=q.dtype, device=q.device, ) @@ -704,7 +704,6 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): kv_c_and_k_pe_cache, chunk_workspace, chunk.block_table, - chunk.seq_lens, chunk.workspace_starts, len(chunk.block_table), ) @@ -738,7 +737,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): # Convert per-request indices to global slots (decode) or workspace # offsets (prefill). topk_indices = triton_convert_req_index_to_global_index( - attn_metadata.req_id_per_token, + attn_metadata.req_id_per_token[: topk_indices.shape[0]], attn_metadata.block_table, topk_indices, BLOCK_SIZE=attn_metadata.block_size, diff --git a/vllm/v1/attention/backends/mla/sparse_utils.py b/vllm/v1/attention/backends/mla/sparse_utils.py index 522b52b0dc..681ab7cfd9 100644 --- a/vllm/v1/attention/backends/mla/sparse_utils.py +++ b/vllm/v1/attention/backends/mla/sparse_utils.py @@ -154,6 +154,11 @@ def triton_convert_req_index_to_global_index( assert req_id.dtype == torch.int32 assert block_table.dtype == torch.int32 assert token_indices.dtype == torch.int32 + assert req_id.shape[0] == token_indices.shape[0], ( + f"req_id ({req_id.shape[0]}) and token_indices ({token_indices.shape[0]}) " + "must cover the same tokens; the grid is sized by req_id but the output " + "is allocated like token_indices, so a longer req_id writes out of bounds" + ) assert token_indices.shape[1] == NUM_TOPK_TOKENS assert NUM_TOPK_TOKENS % BLOCK_N == 0, ( f"NUM_TOPK_TOKENS ({NUM_TOPK_TOKENS}) must be divisible by BLOCK_N ({BLOCK_N})"