From 699e180df48d78b5275d528124641c67b4b1664c Mon Sep 17 00:00:00 2001 From: Gabriel Wu <13583761+lucifer1004@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:44:42 +0800 Subject: [PATCH] [Bugfix][SM120] DSv4: pass contiguous C128A decode topk indices on SM120 (#53574) Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../kernels/attention/test_flashmla_sparse.py | 27 +++++++++++++++---- vllm/models/deepseek_v4/sparse_mla.py | 11 ++++++-- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/tests/kernels/attention/test_flashmla_sparse.py b/tests/kernels/attention/test_flashmla_sparse.py index e180b035c5..fb93557a32 100644 --- a/tests/kernels/attention/test_flashmla_sparse.py +++ b/tests/kernels/attention/test_flashmla_sparse.py @@ -4,8 +4,19 @@ import pytest import torch -def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride(): - from vllm.models.deepseek_v4.sparse_mla import build_c128a_topk_metadata +@pytest.mark.parametrize("sm120", [False, True]) +def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride( + monkeypatch: pytest.MonkeyPatch, + sm120: bool, +): + from vllm.models.deepseek_v4 import sparse_mla + from vllm.platforms.interface import DeviceCapability + + monkeypatch.setattr( + sparse_mla.current_platform, + "get_device_capability", + lambda: DeviceCapability(12, 0) if sm120 else DeviceCapability(10, 0), + ) device = torch.device("cuda") capacity_width = 512 @@ -27,12 +38,18 @@ def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride(): decode_lens_buffer=torch.empty(2, dtype=torch.int32, device=device), prefill_buffer=prefill_buffer, ) - captured_decode, _, captured_prefill = build_c128a_topk_metadata( + captured_decode, _, captured_prefill = sparse_mla.build_c128a_topk_metadata( max_compressed_tokens=256, **kwargs, ) - assert captured_decode.shape == captured_prefill.shape == (2, 256) + # SM120 keeps the decode view contiguous across the full buffer width; + # other backends get the active-width slice. The prefill view is always + # narrowed. + expected_decode_width = capacity_width if sm120 else 256 + assert captured_decode.shape == (2, expected_decode_width) + assert captured_prefill.shape == (2, 256) assert captured_decode.stride(0) == captured_prefill.stride(0) == capacity_width + assert captured_decode.is_contiguous() == sm120 captured_rows = torch.empty((4, 4), dtype=torch.int32, device=device) captured_rows[:2].copy_(captured_decode[:, :4]) @@ -45,7 +62,7 @@ def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride(): global_decode_buffer.fill_(-99) prefill_buffer.fill_(-99) - build_c128a_topk_metadata( + sparse_mla.build_c128a_topk_metadata( max_compressed_tokens=128, **kwargs, ) diff --git a/vllm/models/deepseek_v4/sparse_mla.py b/vllm/models/deepseek_v4/sparse_mla.py index ebff031cea..7f371c3922 100644 --- a/vllm/models/deepseek_v4/sparse_mla.py +++ b/vllm/models/deepseek_v4/sparse_mla.py @@ -9,6 +9,7 @@ import torch from vllm.config import VllmConfig from vllm.config.cache import CacheDType +from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv @@ -310,7 +311,7 @@ def build_c128a_topk_metadata( Prefill tokens: position → local indices [0, ..., n-1, -1, ...]. Writes into pre-allocated buffers for CUDA graph address stability. - Returns slices of the buffers. + Returns views of the buffers. """ num_tokens = positions.shape[0] num_prefill_tokens = num_tokens - num_decode_tokens @@ -322,7 +323,13 @@ def build_c128a_topk_metadata( ) assert global_decode_buffer.stride(-1) == prefill_buffer.stride(-1) == 1 - global_decode = global_decode_buffer[:num_decode_tokens, :max_compressed_tokens] + # TODO: support adaptive-width decode on SM120 (needs the FlashInfer + # SM120 kernel to accept a real row stride for eidx). + capability = current_platform.get_device_capability() + if capability is not None and capability.major == 12: + global_decode = global_decode_buffer[:num_decode_tokens] + else: + global_decode = global_decode_buffer[:num_decode_tokens, :max_compressed_tokens] decode_lens = decode_lens_buffer[:num_decode_tokens] prefill_local = prefill_buffer[:num_prefill_tokens, :max_compressed_tokens] assert global_decode.stride(0) == global_decode_buffer.stride(0)