[Bugfix] Make DSV4 sparse MLA work end-to-end for plain decode, MTP, and DSpark (#51538)
This commit is contained in:
@@ -225,6 +225,7 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
|
||||
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32),
|
||||
decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32),
|
||||
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
|
||||
decode_swa_width=4,
|
||||
is_valid_token=torch.tensor([True], dtype=torch.bool),
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
@@ -331,3 +332,120 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
|
||||
assert builder_calls == 4
|
||||
assert sparse_indices_third is not sparse_indices_fourth
|
||||
assert sparse_lens_third is not sparse_lens_fourth
|
||||
|
||||
|
||||
def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch):
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
captured_shapes_and_windows: list[tuple[int, int]] = []
|
||||
|
||||
def fake_build(*args, **kwargs):
|
||||
# window_size is the 12th positional arg of
|
||||
# build_flashinfer_mixed_sparse_indices.
|
||||
captured_shapes_and_windows.append((args[0].shape[-1], args[11]))
|
||||
num_tokens = args[0].shape[0] + args[3].shape[0]
|
||||
return (
|
||||
torch.zeros((num_tokens, 1), dtype=torch.int32),
|
||||
torch.zeros((num_tokens,), dtype=torch.int32),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
|
||||
)
|
||||
|
||||
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 4
|
||||
attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32)
|
||||
|
||||
wide_width = 8
|
||||
wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32)
|
||||
wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32)
|
||||
wide_metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0], dtype=torch.int32),
|
||||
decode_swa_indices=wide_indices,
|
||||
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
|
||||
decode_swa_width=wide_width,
|
||||
is_valid_token=torch.tensor([True], dtype=torch.bool),
|
||||
num_decodes=1,
|
||||
num_prefills=0,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=0,
|
||||
)
|
||||
attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=wide_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
assert captured_shapes_and_windows == [(wide_width, attn.window_size)]
|
||||
|
||||
empty_width = 8
|
||||
empty_metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 2], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32),
|
||||
decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32),
|
||||
decode_swa_lens=torch.empty((0,), dtype=torch.int32),
|
||||
decode_swa_width=empty_width,
|
||||
is_valid_token=torch.tensor([True, True], dtype=torch.bool),
|
||||
num_decodes=0,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=0,
|
||||
num_prefill_tokens=2,
|
||||
)
|
||||
attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=empty_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
assert captured_shapes_and_windows == [
|
||||
(wide_width, attn.window_size),
|
||||
(empty_width, attn.window_size),
|
||||
]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
padded_width = 8
|
||||
logical_window = 4
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=torch.empty(
|
||||
(0, padded_width), dtype=torch.int32, device=device
|
||||
),
|
||||
decode_compressed_indices=None,
|
||||
decode_compressed_topk_lens=None,
|
||||
prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=None,
|
||||
compressed_block_size=64,
|
||||
window_size=logical_window,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_indices.shape == (1, padded_width)
|
||||
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
|
||||
assert sparse_lens.cpu().tolist() == [padded_width]
|
||||
|
||||
@@ -1095,6 +1095,101 @@ def test_convert_standard_mxfp4_weights_for_flashinfer_cutlass(
|
||||
torch.testing.assert_close(interleaved_scales[1], w2_scale)
|
||||
|
||||
|
||||
def test_gpt_oss_quant_config_supplies_clamped_swiglu_params():
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import GptOssMxfp4MoEMethod
|
||||
|
||||
fake_method = types.SimpleNamespace(
|
||||
mxfp4_backend=Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
w13_precision_config=None,
|
||||
w2_precision_config=None,
|
||||
)
|
||||
fake_layer = types.SimpleNamespace(
|
||||
w13_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
|
||||
w2_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
|
||||
)
|
||||
|
||||
quant_config = GptOssMxfp4MoEMethod.get_fused_moe_quant_config(
|
||||
fake_method, fake_layer
|
||||
)
|
||||
|
||||
assert quant_config is not None
|
||||
assert quant_config.gemm1_alpha == 1.702
|
||||
assert quant_config.gemm1_beta == 1.0
|
||||
assert quant_config.gemm1_clamp_limit == 7.0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="CUDA is required for FlashInferExperts parameter storage",
|
||||
)
|
||||
@pytest.mark.parametrize("supplied", [False, True])
|
||||
def test_flashinfer_experts_swiglu_params_follow_quant_config(supplied: bool):
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
|
||||
num_experts, intermediate_size, hidden_size = 2, 128, 256
|
||||
w1_scale = torch.zeros(
|
||||
(num_experts, 2 * intermediate_size, hidden_size // 32),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
w2_scale = torch.zeros(
|
||||
(num_experts, hidden_size, intermediate_size // 32),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
extra = (
|
||||
{"gemm1_alpha": 0.5, "gemm1_beta": 0.25, "gemm1_clamp_limit": 3.0}
|
||||
if supplied
|
||||
else {}
|
||||
)
|
||||
quant_config = mxfp4_w4a16_moe_quant_config(
|
||||
w1_scale=w1_scale, w2_scale=w2_scale, **extra
|
||||
)
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=1,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
)
|
||||
experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config)
|
||||
|
||||
if supplied:
|
||||
for name, value in (
|
||||
("gemm1_alpha", 0.5),
|
||||
("gemm1_beta", 0.25),
|
||||
("gemm1_clamp_limit", 3.0),
|
||||
):
|
||||
parameter = getattr(experts, name)
|
||||
assert isinstance(parameter, torch.Tensor)
|
||||
assert parameter.dtype == torch.float32
|
||||
assert parameter.shape == (num_experts,)
|
||||
torch.testing.assert_close(
|
||||
parameter.cpu(),
|
||||
torch.full((num_experts,), value, dtype=torch.float32),
|
||||
)
|
||||
else:
|
||||
assert experts.gemm1_alpha is None
|
||||
assert experts.gemm1_beta is None
|
||||
assert experts.gemm1_clamp_limit is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topk", [1, 4])
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 128])
|
||||
|
||||
@@ -31,10 +31,23 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
|
||||
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("window_size", "num_speculative_tokens", "expected"),
|
||||
[(128, 5, 192), (512, 5, 576), (1024, 0, 1024)],
|
||||
)
|
||||
def test_get_dspark_swa_index_width(
|
||||
window_size: int, num_speculative_tokens: int, expected: int
|
||||
):
|
||||
assert get_dspark_swa_index_width(window_size, num_speculative_tokens) == expected
|
||||
|
||||
|
||||
def test_compute_global_topk_reuses_output_buffers():
|
||||
device = "cuda"
|
||||
topk_indices = torch.tensor(
|
||||
|
||||
Reference in New Issue
Block a user