[Bugfix] Make DSV4 sparse MLA work end-to-end for plain decode, MTP, and DSpark (#51538)

This commit is contained in:
Gabriel Wu
2026-08-15 07:01:32 -07:00
committed by GitHub
parent ac2ae8798c
commit 97388c44f9
20 changed files with 797 additions and 120 deletions
@@ -225,6 +225,7 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32),
decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32),
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
decode_swa_width=4,
is_valid_token=torch.tensor([True], dtype=torch.bool),
num_decodes=1,
num_prefills=1,
@@ -331,3 +332,120 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
assert builder_calls == 4
assert sparse_indices_third is not sparse_indices_fourth
assert sparse_lens_third is not sparse_lens_fourth
def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch):
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
captured_shapes_and_windows: list[tuple[int, int]] = []
def fake_build(*args, **kwargs):
# window_size is the 12th positional arg of
# build_flashinfer_mixed_sparse_indices.
captured_shapes_and_windows.append((args[0].shape[-1], args[11]))
num_tokens = args[0].shape[0] + args[3].shape[0]
return (
torch.zeros((num_tokens, 1), dtype=torch.int32),
torch.zeros((num_tokens,), dtype=torch.int32),
)
monkeypatch.setattr(
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
)
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = 1
attn.window_size = 4
attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32)
wide_width = 8
wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32)
wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32)
wide_metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
slot_mapping=torch.tensor([0], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8], dtype=torch.int32),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
token_to_req_indices=torch.tensor([0], dtype=torch.int32),
decode_swa_indices=wide_indices,
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
decode_swa_width=wide_width,
is_valid_token=torch.tensor([True], dtype=torch.bool),
num_decodes=1,
num_prefills=0,
num_decode_tokens=1,
num_prefill_tokens=0,
)
attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=wide_metadata,
attn_metadata=None,
swa_only=True,
)
assert captured_shapes_and_windows == [(wide_width, attn.window_size)]
empty_width = 8
empty_metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8], dtype=torch.int32),
query_start_loc=torch.tensor([0, 2], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32),
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32),
decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32),
decode_swa_lens=torch.empty((0,), dtype=torch.int32),
decode_swa_width=empty_width,
is_valid_token=torch.tensor([True, True], dtype=torch.bool),
num_decodes=0,
num_prefills=1,
num_decode_tokens=0,
num_prefill_tokens=2,
)
attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=empty_metadata,
attn_metadata=None,
swa_only=True,
)
assert captured_shapes_and_windows == [
(wide_width, attn.window_size),
(empty_width, attn.window_size),
]
def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
padded_width = 8
logical_window = 4
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=torch.empty(
(0, padded_width), dtype=torch.int32, device=device
),
decode_compressed_indices=None,
decode_compressed_topk_lens=None,
prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device),
seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device),
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=None,
compressed_block_size=64,
window_size=logical_window,
compress_ratio=1,
topk=0,
)
assert sparse_indices.shape == (1, padded_width)
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
assert sparse_lens.cpu().tolist() == [padded_width]
+95
View File
@@ -1095,6 +1095,101 @@ def test_convert_standard_mxfp4_weights_for_flashinfer_cutlass(
torch.testing.assert_close(interleaved_scales[1], w2_scale)
def test_gpt_oss_quant_config_supplies_clamped_swiglu_params():
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
from vllm.model_executor.layers.quantization.mxfp4 import GptOssMxfp4MoEMethod
fake_method = types.SimpleNamespace(
mxfp4_backend=Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
w13_precision_config=None,
w2_precision_config=None,
)
fake_layer = types.SimpleNamespace(
w13_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
w2_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
)
quant_config = GptOssMxfp4MoEMethod.get_fused_moe_quant_config(
fake_method, fake_layer
)
assert quant_config is not None
assert quant_config.gemm1_alpha == 1.702
assert quant_config.gemm1_beta == 1.0
assert quant_config.gemm1_clamp_limit == 7.0
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="CUDA is required for FlashInferExperts parameter storage",
)
@pytest.mark.parametrize("supplied", [False, True])
def test_flashinfer_experts_swiglu_params_follow_quant_config(supplied: bool):
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
mxfp4_w4a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
num_experts, intermediate_size, hidden_size = 2, 128, 256
w1_scale = torch.zeros(
(num_experts, 2 * intermediate_size, hidden_size // 32),
dtype=torch.uint8,
device="cuda",
)
w2_scale = torch.zeros(
(num_experts, hidden_size, intermediate_size // 32),
dtype=torch.uint8,
device="cuda",
)
extra = (
{"gemm1_alpha": 0.5, "gemm1_beta": 0.25, "gemm1_clamp_limit": 3.0}
if supplied
else {}
)
quant_config = mxfp4_w4a16_moe_quant_config(
w1_scale=w1_scale, w2_scale=w2_scale, **extra
)
moe_config = FusedMoEConfig(
num_experts=num_experts,
experts_per_token=1,
hidden_dim=hidden_size,
intermediate_size=intermediate_size,
num_local_experts=num_experts,
num_logical_experts=num_experts,
activation=MoEActivation.SILU,
device="cuda",
routing_method=RoutingMethodType.TopK,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
)
experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config)
if supplied:
for name, value in (
("gemm1_alpha", 0.5),
("gemm1_beta", 0.25),
("gemm1_clamp_limit", 3.0),
):
parameter = getattr(experts, name)
assert isinstance(parameter, torch.Tensor)
assert parameter.dtype == torch.float32
assert parameter.shape == (num_experts,)
torch.testing.assert_close(
parameter.cpu(),
torch.full((num_experts,), value, dtype=torch.float32),
)
else:
assert experts.gemm1_alpha is None
assert experts.gemm1_beta is None
assert experts.gemm1_clamp_limit is None
@pytest.mark.parametrize("topk", [1, 4])
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("num_tokens", [1, 128])
+13
View File
@@ -31,10 +31,23 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
)
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
@pytest.mark.parametrize(
("window_size", "num_speculative_tokens", "expected"),
[(128, 5, 192), (512, 5, 576), (1024, 0, 1024)],
)
def test_get_dspark_swa_index_width(
window_size: int, num_speculative_tokens: int, expected: int
):
assert get_dspark_swa_index_width(window_size, num_speculative_tokens) == expected
def test_compute_global_topk_reuses_output_buffers():
device = "cuda"
topk_indices = torch.tensor(