From 50d812b66a697c464a9964b3a061b47583b83e34 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Fri, 18 Sep 2026 12:57:19 -0700 Subject: [PATCH] [Perf][DSV4.1] Optimize MegaMoE staging and NVFP4 cache gathers (#57604) Signed-off-by: Woosuk Kwon Co-authored-by: Codex --- tests/kernels/test_compressor_kv_cache.py | 95 +++++++++++++------ tests/models/test_deepseek_v4_mega_moe.py | 22 +++-- .../deepseek_v4/nvidia/ops/prepare_megamoe.py | 67 ++++++++----- .../deepseek_v41/common/ops/cache_utils.py | 6 +- .../nvidia/flash_mla_mega_attn.py | 2 +- 5 files changed, 130 insertions(+), 62 deletions(-) diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py index 95ff6b3c05..06897f04c0 100644 --- a/tests/kernels/test_compressor_kv_cache.py +++ b/tests/kernels/test_compressor_kv_cache.py @@ -473,15 +473,18 @@ _E2M1_MAGNITUDES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: - """Unpack one 256-byte e2m1 row and apply its 32 e4m3 tile scales.""" + """Unpack e2m1 rows and apply their e4m3 tile scales.""" mags = torch.tensor(_E2M1_MAGNITUDES, device=packed.device) - codes = torch.empty(512, dtype=torch.uint8, device=packed.device) - codes[0::2] = packed & 0xF # even element in the low nibble - codes[1::2] = packed >> 4 + codes = torch.empty( + (*packed.shape[:-1], 512), dtype=torch.uint8, device=packed.device + ) + codes[..., 0::2] = packed & 0xF # even element in the low nibble + codes[..., 1::2] = packed >> 4 vals = mags[(codes & 7).long()] * torch.where(codes >= 8, -1.0, 1.0) return ( - vals.view(32, 16) * scales.view(torch.float8_e4m3fn).float().view(32, 1) - ).flatten() + vals.reshape(*packed.shape[:-1], 32, 16) + * scales.view(torch.float8_e4m3fn).float().reshape(*packed.shape[:-1], 32, 1) + ).flatten(-2) @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") @@ -543,7 +546,10 @@ def test_v41_rope_insert_nvfp4_record(compress_ratio: int): @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") -def test_v41_nvfp4_gather_matches_insert(): +@pytest.mark.parametrize("num_reqs", [1, 8, 32]) +@pytest.mark.parametrize("num_tokens", [70, 128, 129, 2051]) +@pytest.mark.parametrize("partial", [False, True]) +def test_v41_nvfp4_gather_matches_insert(num_reqs, num_tokens, partial): """The NVFP4 gather dequantizes exactly what the insert kernel wrote.""" from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import ( @@ -553,38 +559,73 @@ def test_v41_nvfp4_gather_matches_insert(): torch.manual_seed(17) device = "cuda" block_size = 64 - num_tokens = 70 - num_blocks = 4 + num_blocks = math.ceil(num_tokens / block_size) page_bytes = math.ceil(block_size * 288 / 512) * 512 positions = torch.arange(num_tokens, dtype=torch.int64, device=device) latent = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device) - angles = torch.randn(128, 32, device=device) + angles = torch.randn(num_tokens, 32, device=device) cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1) backing = torch.zeros(num_blocks, page_bytes, dtype=torch.uint8, device=device) cache = backing.as_strided((num_blocks, block_size, 288), (page_bytes, 288, 1)) slots = torch.arange(num_tokens, dtype=torch.int64, device=device) rope_quant_insert(latent, positions, cos_sin, cache, slots, 1) - out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device) - dequantize_and_gather_k_cache( - out, - cache, - seq_lens=torch.tensor([num_tokens], dtype=torch.int32, device=device), - gather_lens=None, - block_table=torch.arange(num_blocks, dtype=torch.int32, device=device).view( - 1, -1 - ), - block_size=block_size, - offset=0, + offset = 7 if partial else 0 + seq_lens = num_tokens - torch.arange(num_reqs, dtype=torch.int32, device=device) + gather_lens = seq_lens - 3 if partial else None + storage = torch.full( + (num_reqs, num_tokens + offset + 16, 512), + -123.0, + dtype=torch.bfloat16, + device=device, ) - for t in (0, 1, block_size, num_tokens - 1): - page, row = divmod(t, block_size) - expected = _decode_nvfp4_row( - backing[page, row * 256 : (row + 1) * 256], - backing[page, block_size * 256 + row * 32 :][:32], + out = storage[:, : num_tokens + offset] + block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).repeat( + num_reqs, 1 + ) + + def gather(): + dequantize_and_gather_k_cache( + out, + cache, + seq_lens=seq_lens, + gather_lens=gather_lens, + block_table=block_table, + block_size=block_size, + offset=offset, ) - torch.testing.assert_close(out[0, t].float(), expected, rtol=0, atol=0) + + expected = _decode_nvfp4_row( + backing[:, : block_size * 256].reshape(-1, 256), + backing[:, block_size * 256 : block_size * 288].reshape(-1, 32), + ) + + def check(shorter_by): + assert (storage[:, num_tokens + offset :] == -123).all() + for req in range(num_reqs): + start = 3 if partial else 0 + length = num_tokens - req - start - shorter_by + assert (out[req, :offset] == -123).all() + assert (out[req, offset + length :] == -123).all() + torch.testing.assert_close( + out[req, offset : offset + length].float(), + expected[start : start + length], + rtol=0, + atol=0, + ) + + gather() + check(0) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + gather() + storage.fill_(-123) + seq_lens.sub_(1) + if gather_lens is not None: + gather_lens.sub_(1) + graph.replay() + check(1) @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") diff --git a/tests/models/test_deepseek_v4_mega_moe.py b/tests/models/test_deepseek_v4_mega_moe.py index ad029160c5..f87dd75fb2 100644 --- a/tests/models/test_deepseek_v4_mega_moe.py +++ b/tests/models/test_deepseek_v4_mega_moe.py @@ -686,12 +686,14 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert( not torch.cuda.is_available(), reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.", ) -def test_deepseek_v4_mega_moe_fused_input_staging_is_bitwise_exact(): +@pytest.mark.parametrize("num_tokens", [7, 63, 64, 65, 127, 128, 135, 434, 16384]) +@pytest.mark.parametrize("hidden_size", [256, 5120]) +def test_deepseek_v4_mega_moe_fused_input_staging_is_bitwise_exact( + num_tokens, hidden_size +): from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8 device = torch.device("cuda") - num_tokens = 7 - hidden_size = 256 top_k = 8 generator = torch.Generator(device=device) @@ -846,6 +848,7 @@ def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m): assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8)) assert torch.equal(fused_x_sf, ref_x_sf) assert torch.equal(fused_shared_x_sf[populated], ref_shared_x_sf[populated]) + assert torch.all(fused_shared_x_sf[~populated] == -1) def test_deepseek_v4_pwal_hook_finalizes_mega_moe_and_mhc_broadcast(): @@ -884,11 +887,13 @@ def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe(): reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.", ) @pytest.mark.parametrize("nonfinite_padding", [False, True]) -def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(nonfinite_padding): +@pytest.mark.parametrize("num_tokens", [7, 65, 135]) +def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding( + nonfinite_padding, num_tokens +): from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8 device = torch.device("cuda") - num_tokens = 7 hidden_size = 256 top_k = 8 @@ -916,10 +921,9 @@ def test_deepseek_v4_mega_moe_fused_input_staging_masks_padding(nonfinite_paddin dtype=torch.float32, generator=generator, ) - is_padding = torch.tensor( - [False, True, False, False, True, False, True], - device=device, - ) + is_padding = torch.zeros(num_tokens, device=device, dtype=torch.bool) + is_padding[1::3] = True + is_padding[-1] = True if nonfinite_padding: topk_weights[is_padding] = float("nan") diff --git a/vllm/models/deepseek_v4/nvidia/ops/prepare_megamoe.py b/vllm/models/deepseek_v4/nvidia/ops/prepare_megamoe.py index d254dad9d6..aa9482bed6 100644 --- a/vllm/models/deepseek_v4/nvidia/ops/prepare_megamoe.py +++ b/vllm/models/deepseek_v4/nvidia/ops/prepare_megamoe.py @@ -12,7 +12,7 @@ import torch from vllm.triton_utils import tl, triton -@triton.jit +@triton.jit(do_not_specialize=["num_tokens"]) def _prepare_megamoe_inputs_kernel( hidden_states, x_fp8, @@ -40,27 +40,32 @@ def _prepare_megamoe_inputs_kernel( topk_idx_stride_k: tl.constexpr, topk_weights_out_stride_m: tl.constexpr, topk_weights_out_stride_k: tl.constexpr, + num_tokens, hidden_size: tl.constexpr, top_k: tl.constexpr, + BLOCK_M: tl.constexpr, BLOCK_K: tl.constexpr, GROUP_K: tl.constexpr, BLOCK_TOPK: tl.constexpr, SHARED_BLOCK_M: tl.constexpr, ) -> None: - token_id = tl.program_id(0) + token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) + token_mask = token_id < num_tokens k_block_id = tl.program_id(1) k_offsets = k_block_id * BLOCK_K + tl.arange(0, BLOCK_K) - k_mask = k_offsets < hidden_size + k_mask = token_mask[:, None] & (k_offsets[None, :] < hidden_size) hidden = tl.load( - hidden_states + token_id * hidden_stride_m + k_offsets * hidden_stride_k, + hidden_states + + token_id[:, None] * hidden_stride_m + + k_offsets[None, :] * hidden_stride_k, mask=k_mask, other=0.0, ).to(tl.float32) num_groups: tl.constexpr = BLOCK_K // GROUP_K - hidden_groups = tl.reshape(tl.abs(hidden), [num_groups, GROUP_K]) - amax = tl.max(hidden_groups, axis=1) + hidden_groups = tl.reshape(tl.abs(hidden), [BLOCK_M, num_groups, GROUP_K]) + amax = tl.max(hidden_groups, axis=2) amax = tl.maximum(amax, 1.0e-4) scale = amax / 448.0 @@ -71,21 +76,24 @@ def _prepare_megamoe_inputs_kernel( scale_exp = tl.minimum(tl.maximum(scale_exp, 1), 254) rounded_scale = (scale_exp << 23).to(tl.float32, bitcast=True) - hidden_groups = tl.reshape(hidden, [num_groups, GROUP_K]) - scaled = hidden_groups * (1.0 / rounded_scale)[:, None] - scaled = tl.reshape(scaled, [BLOCK_K]) + hidden_groups = tl.reshape(hidden, [BLOCK_M, num_groups, GROUP_K]) + scaled = hidden_groups * (1.0 / rounded_scale)[:, :, None] + scaled = tl.reshape(scaled, [BLOCK_M, BLOCK_K]) fp8 = scaled.to(tl.float8e4nv) tl.store( - x_fp8 + token_id * x_stride_m + k_offsets * x_stride_k, + x_fp8 + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k, fp8, mask=k_mask, ) scale_offsets = tl.arange(0, num_groups) - packed_scale = tl.sum(scale_exp << (scale_offsets * 8), axis=0).to(tl.int32) + packed_scale = tl.sum(scale_exp << (scale_offsets[None, :] * 8), axis=1).to( + tl.int32 + ) tl.store( x_sf + token_id * x_sf_stride_m + k_block_id * x_sf_stride_k, packed_scale, + mask=token_mask, ) # DeepGEMM's SM100 shared-expert TMA loads require the activation scales @@ -105,41 +113,48 @@ def _prepare_megamoe_inputs_kernel( + shared_row * shared_x_sf_stride_m + k_block_id * shared_x_sf_stride_k, packed_scale, + mask=token_mask, ) if k_block_id == 0: topk_offsets = tl.arange(0, BLOCK_TOPK) - topk_mask = topk_offsets < top_k - token_is_padding = False + topk_mask = token_mask[:, None] & (topk_offsets[None, :] < top_k) + token_is_padding = tl.full((BLOCK_M,), False, tl.int1) if is_padding is not None: - token_is_padding = tl.load(is_padding + token_id * is_padding_stride_m) + token_is_padding = tl.load( + is_padding + token_id * is_padding_stride_m, + mask=token_mask, + other=True, + ) ids = tl.load( - topk_ids + token_id * topk_ids_stride_m + topk_offsets * topk_ids_stride_k, + topk_ids + + token_id[:, None] * topk_ids_stride_m + + topk_offsets[None, :] * topk_ids_stride_k, mask=topk_mask, other=0, ).to(tl.int64) - ids = tl.where(token_is_padding, -1, ids) + ids = tl.where(token_is_padding[:, None], -1, ids) tl.store( topk_idx_out - + token_id * topk_idx_stride_m - + topk_offsets * topk_idx_stride_k, + + token_id[:, None] * topk_idx_stride_m + + topk_offsets[None, :] * topk_idx_stride_k, ids, mask=topk_mask, ) weights = tl.load( topk_weights - + token_id * topk_weights_stride_m - + topk_offsets * topk_weights_stride_k, + + token_id[:, None] * topk_weights_stride_m + + topk_offsets[None, :] * topk_weights_stride_k, mask=topk_mask, other=0.0, ) - weights = tl.where(token_is_padding, 0.0, weights) + weights = tl.where(token_is_padding[:, None], 0.0, weights) tl.store( topk_weights_out - + token_id * topk_weights_out_stride_m - + topk_offsets * topk_weights_out_stride_k, + + token_id[:, None] * topk_weights_out_stride_m + + topk_offsets[None, :] * topk_weights_out_stride_k, weights, mask=topk_mask, ) @@ -195,7 +210,9 @@ def prepare_megamoe_inputs( ) block_k = 128 - grid = (num_tokens, triton.cdiv(hidden_size, block_k)) + # On GB200, eight-row tiles win from 64 tokens; keep smaller batches untiled. + block_m = 8 if num_tokens >= 64 else 1 + grid = (triton.cdiv(num_tokens, block_m), triton.cdiv(hidden_size, block_k)) block_topk = triton.next_power_of_2(top_k) padding_stride_m = is_padding.stride(0) if is_padding is not None else 0 _prepare_megamoe_inputs_kernel[grid]( @@ -225,8 +242,10 @@ def prepare_megamoe_inputs( topk_idx_out.stride(1), topk_weights_out.stride(0), topk_weights_out.stride(1), + num_tokens, hidden_size, top_k, + BLOCK_M=block_m, BLOCK_K=block_k, GROUP_K=32, BLOCK_TOPK=block_topk, diff --git a/vllm/models/deepseek_v41/common/ops/cache_utils.py b/vllm/models/deepseek_v41/common/ops/cache_utils.py index e7c3650768..04b2a9bac1 100644 --- a/vllm/models/deepseek_v41/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v41/common/ops/cache_utils.py @@ -584,7 +584,11 @@ def dequantize_and_gather_k_cache_triton( num_reqs = seq_lens.shape[0] NUM_WORKERS = 128 if k_cache.shape[-1] == V41_NVFP4_BYTES_PER_TOKEN: - _dequantize_and_gather_k_nvfp4_kernel[(num_reqs, NUM_WORKERS)]( + # Keep short gathers unchanged; cap total workers for multi-request chunks. + num_workers = min( + max(128, 2048 // max(1, num_reqs)), max(128, out.shape[1] - offset) + ) + _dequantize_and_gather_k_nvfp4_kernel[(num_reqs, num_workers)]( out, out.stride(0), out.stride(1), diff --git a/vllm/models/deepseek_v41/nvidia/flash_mla_mega_attn.py b/vllm/models/deepseek_v41/nvidia/flash_mla_mega_attn.py index 9722673dd1..6f4dc7e96c 100644 --- a/vllm/models/deepseek_v41/nvidia/flash_mla_mega_attn.py +++ b/vllm/models/deepseek_v41/nvidia/flash_mla_mega_attn.py @@ -422,7 +422,7 @@ class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention): if not swa_only: assert flashmla_metadata is not None dequantize_and_gather_k_cache( - kv_ws[:chunk_size], + kv_ws[:chunk_size, :chunk_n], self._compressed_kv_cache(), seq_lens=seq_lens[chunk_start:chunk_end] // self.compress_ratio, gather_lens=None,