[Bugfix][SM120] DSv4: pass contiguous C128A decode topk indices on SM120 (#53574)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Wentao Ye
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
28bf75c9a9
commit
699e180df4
@@ -4,8 +4,19 @@ import pytest
|
||||
import torch
|
||||
|
||||
|
||||
def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride():
|
||||
from vllm.models.deepseek_v4.sparse_mla import build_c128a_topk_metadata
|
||||
@pytest.mark.parametrize("sm120", [False, True])
|
||||
def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sm120: bool,
|
||||
):
|
||||
from vllm.models.deepseek_v4 import sparse_mla
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mla.current_platform,
|
||||
"get_device_capability",
|
||||
lambda: DeviceCapability(12, 0) if sm120 else DeviceCapability(10, 0),
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
capacity_width = 512
|
||||
@@ -27,12 +38,18 @@ def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride():
|
||||
decode_lens_buffer=torch.empty(2, dtype=torch.int32, device=device),
|
||||
prefill_buffer=prefill_buffer,
|
||||
)
|
||||
captured_decode, _, captured_prefill = build_c128a_topk_metadata(
|
||||
captured_decode, _, captured_prefill = sparse_mla.build_c128a_topk_metadata(
|
||||
max_compressed_tokens=256,
|
||||
**kwargs,
|
||||
)
|
||||
assert captured_decode.shape == captured_prefill.shape == (2, 256)
|
||||
# SM120 keeps the decode view contiguous across the full buffer width;
|
||||
# other backends get the active-width slice. The prefill view is always
|
||||
# narrowed.
|
||||
expected_decode_width = capacity_width if sm120 else 256
|
||||
assert captured_decode.shape == (2, expected_decode_width)
|
||||
assert captured_prefill.shape == (2, 256)
|
||||
assert captured_decode.stride(0) == captured_prefill.stride(0) == capacity_width
|
||||
assert captured_decode.is_contiguous() == sm120
|
||||
|
||||
captured_rows = torch.empty((4, 4), dtype=torch.int32, device=device)
|
||||
captured_rows[:2].copy_(captured_decode[:, :4])
|
||||
@@ -45,7 +62,7 @@ def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride():
|
||||
|
||||
global_decode_buffer.fill_(-99)
|
||||
prefill_buffer.fill_(-99)
|
||||
build_c128a_topk_metadata(
|
||||
sparse_mla.build_c128a_topk_metadata(
|
||||
max_compressed_tokens=128,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -310,7 +311,7 @@ def build_c128a_topk_metadata(
|
||||
Prefill tokens: position → local indices [0, ..., n-1, -1, ...].
|
||||
|
||||
Writes into pre-allocated buffers for CUDA graph address stability.
|
||||
Returns slices of the buffers.
|
||||
Returns views of the buffers.
|
||||
"""
|
||||
num_tokens = positions.shape[0]
|
||||
num_prefill_tokens = num_tokens - num_decode_tokens
|
||||
@@ -322,7 +323,13 @@ def build_c128a_topk_metadata(
|
||||
)
|
||||
assert global_decode_buffer.stride(-1) == prefill_buffer.stride(-1) == 1
|
||||
|
||||
global_decode = global_decode_buffer[:num_decode_tokens, :max_compressed_tokens]
|
||||
# TODO: support adaptive-width decode on SM120 (needs the FlashInfer
|
||||
# SM120 kernel to accept a real row stride for eidx).
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is not None and capability.major == 12:
|
||||
global_decode = global_decode_buffer[:num_decode_tokens]
|
||||
else:
|
||||
global_decode = global_decode_buffer[:num_decode_tokens, :max_compressed_tokens]
|
||||
decode_lens = decode_lens_buffer[:num_decode_tokens]
|
||||
prefill_local = prefill_buffer[:num_prefill_tokens, :max_compressed_tokens]
|
||||
assert global_decode.stride(0) == global_decode_buffer.stride(0)
|
||||
|
||||
Reference in New Issue
Block a user