[Bugfix][SM120] DSv4: pass contiguous C128A decode topk indices on SM120 (#53574)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Gabriel Wu
2026-08-31 09:44:42 +00:00
committed by GitHub
co-authored by Wentao Ye mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 28bf75c9a9
commit 699e180df4
2 changed files with 31 additions and 7 deletions
@@ -4,8 +4,19 @@ import pytest
import torch
def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride():
from vllm.models.deepseek_v4.sparse_mla import build_c128a_topk_metadata
@pytest.mark.parametrize("sm120", [False, True])
def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride(
monkeypatch: pytest.MonkeyPatch,
sm120: bool,
):
from vllm.models.deepseek_v4 import sparse_mla
from vllm.platforms.interface import DeviceCapability
monkeypatch.setattr(
sparse_mla.current_platform,
"get_device_capability",
lambda: DeviceCapability(12, 0) if sm120 else DeviceCapability(10, 0),
)
device = torch.device("cuda")
capacity_width = 512
@@ -27,12 +38,18 @@ def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride():
decode_lens_buffer=torch.empty(2, dtype=torch.int32, device=device),
prefill_buffer=prefill_buffer,
)
captured_decode, _, captured_prefill = build_c128a_topk_metadata(
captured_decode, _, captured_prefill = sparse_mla.build_c128a_topk_metadata(
max_compressed_tokens=256,
**kwargs,
)
assert captured_decode.shape == captured_prefill.shape == (2, 256)
# SM120 keeps the decode view contiguous across the full buffer width;
# other backends get the active-width slice. The prefill view is always
# narrowed.
expected_decode_width = capacity_width if sm120 else 256
assert captured_decode.shape == (2, expected_decode_width)
assert captured_prefill.shape == (2, 256)
assert captured_decode.stride(0) == captured_prefill.stride(0) == capacity_width
assert captured_decode.is_contiguous() == sm120
captured_rows = torch.empty((4, 4), dtype=torch.int32, device=device)
captured_rows[:2].copy_(captured_decode[:, :4])
@@ -45,7 +62,7 @@ def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride():
global_decode_buffer.fill_(-99)
prefill_buffer.fill_(-99)
build_c128a_topk_metadata(
sparse_mla.build_c128a_topk_metadata(
max_compressed_tokens=128,
**kwargs,
)
+9 -2
View File
@@ -9,6 +9,7 @@ import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
@@ -310,7 +311,7 @@ def build_c128a_topk_metadata(
Prefill tokens: position → local indices [0, ..., n-1, -1, ...].
Writes into pre-allocated buffers for CUDA graph address stability.
Returns slices of the buffers.
Returns views of the buffers.
"""
num_tokens = positions.shape[0]
num_prefill_tokens = num_tokens - num_decode_tokens
@@ -322,7 +323,13 @@ def build_c128a_topk_metadata(
)
assert global_decode_buffer.stride(-1) == prefill_buffer.stride(-1) == 1
global_decode = global_decode_buffer[:num_decode_tokens, :max_compressed_tokens]
# TODO: support adaptive-width decode on SM120 (needs the FlashInfer
# SM120 kernel to accept a real row stride for eidx).
capability = current_platform.get_device_capability()
if capability is not None and capability.major == 12:
global_decode = global_decode_buffer[:num_decode_tokens]
else:
global_decode = global_decode_buffer[:num_decode_tokens, :max_compressed_tokens]
decode_lens = decode_lens_buffer[:num_decode_tokens]
prefill_local = prefill_buffer[:num_prefill_tokens, :max_compressed_tokens]
assert global_decode.stride(0) == global_decode_buffer.stride(0)