[Bugfix] Make DSV4 sparse MLA work end-to-end for plain decode, MTP, and DSpark (#51538)

This commit is contained in:
Gabriel Wu
2026-08-15 07:01:32 -07:00
committed by GitHub
parent ac2ae8798c
commit 97388c44f9
20 changed files with 797 additions and 120 deletions
@@ -225,6 +225,7 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32),
decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32),
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
decode_swa_width=4,
is_valid_token=torch.tensor([True], dtype=torch.bool),
num_decodes=1,
num_prefills=1,
@@ -331,3 +332,120 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
assert builder_calls == 4
assert sparse_indices_third is not sparse_indices_fourth
assert sparse_lens_third is not sparse_lens_fourth
def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch):
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
captured_shapes_and_windows: list[tuple[int, int]] = []
def fake_build(*args, **kwargs):
# window_size is the 12th positional arg of
# build_flashinfer_mixed_sparse_indices.
captured_shapes_and_windows.append((args[0].shape[-1], args[11]))
num_tokens = args[0].shape[0] + args[3].shape[0]
return (
torch.zeros((num_tokens, 1), dtype=torch.int32),
torch.zeros((num_tokens,), dtype=torch.int32),
)
monkeypatch.setattr(
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
)
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = 1
attn.window_size = 4
attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32)
wide_width = 8
wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32)
wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32)
wide_metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
slot_mapping=torch.tensor([0], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8], dtype=torch.int32),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
token_to_req_indices=torch.tensor([0], dtype=torch.int32),
decode_swa_indices=wide_indices,
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
decode_swa_width=wide_width,
is_valid_token=torch.tensor([True], dtype=torch.bool),
num_decodes=1,
num_prefills=0,
num_decode_tokens=1,
num_prefill_tokens=0,
)
attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=wide_metadata,
attn_metadata=None,
swa_only=True,
)
assert captured_shapes_and_windows == [(wide_width, attn.window_size)]
empty_width = 8
empty_metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8], dtype=torch.int32),
query_start_loc=torch.tensor([0, 2], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32),
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32),
decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32),
decode_swa_lens=torch.empty((0,), dtype=torch.int32),
decode_swa_width=empty_width,
is_valid_token=torch.tensor([True, True], dtype=torch.bool),
num_decodes=0,
num_prefills=1,
num_decode_tokens=0,
num_prefill_tokens=2,
)
attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=empty_metadata,
attn_metadata=None,
swa_only=True,
)
assert captured_shapes_and_windows == [
(wide_width, attn.window_size),
(empty_width, attn.window_size),
]
def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
padded_width = 8
logical_window = 4
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=torch.empty(
(0, padded_width), dtype=torch.int32, device=device
),
decode_compressed_indices=None,
decode_compressed_topk_lens=None,
prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device),
seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device),
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=None,
compressed_block_size=64,
window_size=logical_window,
compress_ratio=1,
topk=0,
)
assert sparse_indices.shape == (1, padded_width)
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
assert sparse_lens.cpu().tolist() == [padded_width]
+95
View File
@@ -1095,6 +1095,101 @@ def test_convert_standard_mxfp4_weights_for_flashinfer_cutlass(
torch.testing.assert_close(interleaved_scales[1], w2_scale)
def test_gpt_oss_quant_config_supplies_clamped_swiglu_params():
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
from vllm.model_executor.layers.quantization.mxfp4 import GptOssMxfp4MoEMethod
fake_method = types.SimpleNamespace(
mxfp4_backend=Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
w13_precision_config=None,
w2_precision_config=None,
)
fake_layer = types.SimpleNamespace(
w13_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
w2_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
)
quant_config = GptOssMxfp4MoEMethod.get_fused_moe_quant_config(
fake_method, fake_layer
)
assert quant_config is not None
assert quant_config.gemm1_alpha == 1.702
assert quant_config.gemm1_beta == 1.0
assert quant_config.gemm1_clamp_limit == 7.0
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="CUDA is required for FlashInferExperts parameter storage",
)
@pytest.mark.parametrize("supplied", [False, True])
def test_flashinfer_experts_swiglu_params_follow_quant_config(supplied: bool):
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
mxfp4_w4a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
num_experts, intermediate_size, hidden_size = 2, 128, 256
w1_scale = torch.zeros(
(num_experts, 2 * intermediate_size, hidden_size // 32),
dtype=torch.uint8,
device="cuda",
)
w2_scale = torch.zeros(
(num_experts, hidden_size, intermediate_size // 32),
dtype=torch.uint8,
device="cuda",
)
extra = (
{"gemm1_alpha": 0.5, "gemm1_beta": 0.25, "gemm1_clamp_limit": 3.0}
if supplied
else {}
)
quant_config = mxfp4_w4a16_moe_quant_config(
w1_scale=w1_scale, w2_scale=w2_scale, **extra
)
moe_config = FusedMoEConfig(
num_experts=num_experts,
experts_per_token=1,
hidden_dim=hidden_size,
intermediate_size=intermediate_size,
num_local_experts=num_experts,
num_logical_experts=num_experts,
activation=MoEActivation.SILU,
device="cuda",
routing_method=RoutingMethodType.TopK,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
)
experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config)
if supplied:
for name, value in (
("gemm1_alpha", 0.5),
("gemm1_beta", 0.25),
("gemm1_clamp_limit", 3.0),
):
parameter = getattr(experts, name)
assert isinstance(parameter, torch.Tensor)
assert parameter.dtype == torch.float32
assert parameter.shape == (num_experts,)
torch.testing.assert_close(
parameter.cpu(),
torch.full((num_experts,), value, dtype=torch.float32),
)
else:
assert experts.gemm1_alpha is None
assert experts.gemm1_beta is None
assert experts.gemm1_clamp_limit is None
@pytest.mark.parametrize("topk", [1, 4])
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("num_tokens", [1, 128])
+13
View File
@@ -31,10 +31,23 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
)
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
@pytest.mark.parametrize(
("window_size", "num_speculative_tokens", "expected"),
[(128, 5, 192), (512, 5, 576), (1024, 0, 1024)],
)
def test_get_dspark_swa_index_width(
window_size: int, num_speculative_tokens: int, expected: int
):
assert get_dspark_swa_index_width(window_size, num_speculative_tokens) == expected
def test_compute_global_topk_reuses_output_buffers():
device = "cuda"
topk_indices = torch.tensor(
@@ -7,6 +7,9 @@ from types import SimpleNamespace
import torch
from vllm.config import set_current_vllm_config
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
_required_sm120_sparse_topk,
)
from vllm.platforms.interface import DeviceCapability
from vllm.utils import flashinfer as fi_utils
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
@@ -52,3 +55,33 @@ def test_v32_glm_sm120_backend_accepts_glm_block_size(
)
assert invalid_reasons == []
def test_sm120_dsv4_capability_checks_exact_dispatch_shape(monkeypatch) -> None:
fake_module = SimpleNamespace(
_DECODE_DSV4_DISPATCH=frozenset({(32, 128), (32, 192)})
)
monkeypatch.setattr(fi_utils, "has_flashinfer_sparse_mla_sm120", lambda: True)
monkeypatch.setattr(fi_utils, "_get_submodule", lambda _name: fake_module)
fi_utils.has_flashinfer_sparse_mla_sm120_config.cache_clear()
assert fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 128)
assert fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 192)
assert not fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 256)
assert not fi_utils.has_flashinfer_sparse_mla_sm120_config(16, 192)
fi_utils.has_flashinfer_sparse_mla_sm120_config.cache_clear()
def test_sm120_dsv4_required_topk_tracks_dspark_width() -> None:
causal = SimpleNamespace(
attention_config=SimpleNamespace(use_non_causal=False),
speculative_config=SimpleNamespace(num_speculative_tokens=5),
)
dspark = SimpleNamespace(
attention_config=SimpleNamespace(use_non_causal=True),
speculative_config=SimpleNamespace(num_speculative_tokens=5),
)
assert _required_sm120_sparse_topk(causal, 128) == 128
assert _required_sm120_sparse_topk(dspark, 128) == 192
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
prepare_dflash_inputs,
)
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires a CUDA device"
)
def _run_prepare(*, target_positions: list[int], block_table_values: list[int]):
device = torch.device("cuda")
max_num_reqs = 4
max_num_tokens = 16
num_speculative_steps = 3
input_buffers = SimpleNamespace(
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
query_start_loc=torch.full(
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
),
seq_lens=torch.full((max_num_reqs,), -1, dtype=torch.int32, device=device),
)
input_batch = SimpleNamespace(
num_reqs=1,
num_scheduled_tokens=np.array([4], dtype=np.int32),
positions=torch.tensor(target_positions, dtype=torch.int64, device=device),
query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device),
idx_mapping=torch.tensor([2], dtype=torch.int32, device=device),
)
query_slot_mapping = torch.full(
(max_num_tokens,), -2, dtype=torch.int64, device=device
)
context_positions = torch.full(
(max_num_tokens,), -1, dtype=torch.int64, device=device
)
context_slot_mapping = torch.full(
(max_num_tokens,), -2, dtype=torch.int64, device=device
)
sample_indices = torch.full(
(max_num_reqs * num_speculative_steps,),
-1,
dtype=torch.int64,
device=device,
)
sample_pos = torch.full_like(sample_indices, -1)
sample_idx_mapping = torch.full(
sample_indices.shape, -1, dtype=torch.int32, device=device
)
temperature = torch.zeros(max_num_reqs, dtype=torch.float32, device=device)
seeds = torch.zeros(max_num_reqs, dtype=torch.int64, device=device)
input_temperature = torch.tensor(
[0.0, 0.0, 1.0, 0.0], dtype=torch.float32, device=device
)
input_seeds = torch.tensor([0, 0, 17, 0], dtype=torch.int64, device=device)
last_sampled = torch.tensor([0, 0, 99, 0], dtype=torch.int64, device=device)
next_prefill_tokens = torch.zeros_like(last_sampled)
block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device)
prepare_dflash_inputs(
input_buffers,
query_slot_mapping,
context_positions,
context_slot_mapping,
sample_indices,
sample_pos,
sample_idx_mapping,
temperature,
seeds,
input_batch,
torch.tensor([1], dtype=torch.int32, device=device),
torch.tensor([2], dtype=torch.int32, device=device),
last_sampled,
next_prefill_tokens,
input_temperature,
input_seeds,
block_table,
4,
123,
num_speculative_steps,
num_speculative_steps,
max_num_reqs,
max_num_tokens,
128,
sample_from_anchor=True,
)
torch.accelerator.synchronize()
return SimpleNamespace(
input_buffers=input_buffers,
query_slot_mapping=query_slot_mapping.cpu(),
context_positions=context_positions.cpu(),
context_slot_mapping=context_slot_mapping.cpu(),
sample_indices=sample_indices.cpu(),
sample_pos=sample_pos.cpu(),
sample_idx_mapping=sample_idx_mapping.cpu(),
temperature=temperature.cpu(),
seeds=seeds.cpu(),
)
def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
# Positions 10/11 use physical block 7. Rejected positions 12/13 would use
# block 8, but must be PAD context rather than contaminating draft KV.
out = _run_prepare(
target_positions=[10, 11, 12, 13],
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
)
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
assert out.context_slot_mapping[:4].tolist() == [30, 31, PAD_SLOT_ID, PAD_SLOT_ID]
# The replacement query starts immediately after the two valid rows and
# advances from the last accepted position (11).
assert out.input_buffers.input_ids[:3].cpu().tolist() == [99, 123, 123]
assert out.input_buffers.positions[:3].cpu().tolist() == [12, 13, 14]
assert out.query_slot_mapping[:3].tolist() == [32, 33, 34]
assert out.sample_indices[:3].tolist() == [0, 1, 2]
assert out.sample_pos[:3].tolist() == [13, 14, 15]
assert out.sample_idx_mapping[:3].tolist() == [2, 2, 2]
assert out.temperature[2].item() == 1.0
assert out.seeds[2].item() == 17
def test_prepare_dflash_inputs_never_writes_the_null_block():
# The valid context uses logical block 0 and the replacement query uses
# logical block 1. Both map to the null block and must remain unwritable.
out = _run_prepare(
target_positions=[2, 3, 4, 5],
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
)
assert out.context_slot_mapping[:4].tolist() == [
PAD_SLOT_ID,
PAD_SLOT_ID,
PAD_SLOT_ID,
PAD_SLOT_ID,
]
assert out.query_slot_mapping[:3].tolist() == [
PAD_SLOT_ID,
PAD_SLOT_ID,
PAD_SLOT_ID,
]
+102
View File
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import pytest
import torch
import vllm.v1.worker.workspace as workspace
from vllm.config import VllmConfig
from vllm.v1.worker.gpu_worker import _num_workspace_lanes
class _SpecConfig:
def __init__(self, dspark: bool) -> None:
self._dspark = dspark
def use_dspark(self) -> bool:
return self._dspark
class _VllmConfig:
def __init__(self, spec_config: _SpecConfig | None) -> None:
self.speculative_config = spec_config
@pytest.mark.parametrize(
("use_v2_model_runner", "spec_config", "expected"),
[
(True, _SpecConfig(True), 2),
(False, _SpecConfig(True), 1),
(True, _SpecConfig(False), 1),
(True, None, 1),
],
)
def test_workspace_lane_count_is_dspark_only(
use_v2_model_runner: bool,
spec_config: _SpecConfig | None,
expected: int,
) -> None:
config = cast(VllmConfig, _VllmConfig(spec_config))
assert _num_workspace_lanes(config, use_v2_model_runner) == expected
def test_workspace_lanes_do_not_alias_and_restore_context(monkeypatch) -> None:
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
manager = workspace.WorkspaceManager(
torch.device("cpu"), num_ubatches=2, num_lanes=2
)
assert manager._current_workspaces == [None, None, None, None]
(target,) = manager.get_simultaneous(((512,), torch.uint8))
with workspace.use_workspace_lane(1):
(draft,) = manager.get_simultaneous(((256,), torch.uint8))
(draft_reused,) = manager.get_simultaneous(((8,), torch.uint8))
(target_reused,) = manager.get_simultaneous(((8,), torch.uint8))
assert manager._current_workspaces[0].numel() == 512 # type: ignore[union-attr]
assert manager._current_workspaces[1].numel() == 256 # type: ignore[union-attr]
assert manager._current_workspaces[2:] == [None, None]
assert target.data_ptr() != draft.data_ptr()
assert draft.data_ptr() == draft_reused.data_ptr()
assert target.data_ptr() == target_reused.data_ptr()
def test_workspace_lanes_compose_with_ubatches(monkeypatch) -> None:
active_ubatch = [0]
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: active_ubatch[0])
manager = workspace.WorkspaceManager(
torch.device("cpu"), num_ubatches=2, num_lanes=2
)
pointers = set()
for ubatch_id in range(2):
active_ubatch[0] = ubatch_id
for lane in range(2):
with workspace.use_workspace_lane(lane):
(buffer,) = manager.get_simultaneous(((16,), torch.uint8))
pointers.add(buffer.data_ptr())
assert len(pointers) == 4
def test_workspace_lane_validation(monkeypatch) -> None:
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1)
with (
pytest.raises(ValueError, match="non-negative"),
workspace.use_workspace_lane(-1),
):
pass
with (
workspace.use_workspace_lane(1),
pytest.raises(RuntimeError, match="is not configured"),
):
manager.get_simultaneous(((1,), torch.uint8))
with pytest.raises(ValueError, match="at least one"):
workspace.WorkspaceManager(torch.device("cpu"), num_lanes=0)