[New model][Multimodal] Add DeepSeek-V4-Flash-Vision-Exp support (#54566)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-02 11:09:09 -07:00
committed by GitHub
parent 488e6fd53c
commit 1356635d83
37 changed files with 2918 additions and 152 deletions
+31
View File
@@ -465,3 +465,34 @@ def test_draft_model_arch_config(
_assert_model_config_methods(
model_config, expected, check_head_size=check_head_size
)
def test_deepseek_v4_convertor_splits_vision_architecture():
"""The vision checkpoint shares model_type/architectures with the
text-only DeepSeek-V4; the convertor routes it to the VL wrapper class
by rewriting hf_config.architectures (model-class resolution reads the
raw attribute)."""
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
from vllm.transformers_utils.model_arch_config_convertor import (
DeepseekV4ModelArchConfigConvertor,
)
text_cfg = DeepseekV4Config(architectures=["DeepseekV4ForCausalLM"])
conv = DeepseekV4ModelArchConfigConvertor(text_cfg, text_cfg)
assert conv.get_architectures() == ["DeepseekV4ForCausalLM"]
assert text_cfg.architectures == ["DeepseekV4ForCausalLM"]
vl_cfg = DeepseekV4Config(
architectures=["DeepseekV4ForCausalLM"], vision_n_layers=32
)
conv = DeepseekV4ModelArchConfigConvertor(vl_cfg, vl_cfg)
assert conv.get_architectures() == ["DeepseekV4ForConditionalGeneration"]
assert vl_cfg.architectures == ["DeepseekV4ForConditionalGeneration"]
# Speculative-draft configs (set by SpeculativeConfig.hf_config_override)
# keep their own architecture even with a vision tower in the config.
for draft_arch in ("DSparkDraftModel", "DeepSeekV4MTPModel"):
draft_cfg = DeepseekV4Config(architectures=[draft_arch], vision_n_layers=32)
conv = DeepseekV4ModelArchConfigConvertor(draft_cfg, draft_cfg)
assert conv.get_architectures() == [draft_arch]
assert draft_cfg.architectures == [draft_arch]
+257 -5
View File
@@ -26,18 +26,46 @@ def _torch_topk_softplus_sqrt(
e_score_correction_bias: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
):
scores = F.softplus(gating_output.float()).sqrt()
original_scores = scores
if e_score_correction_bias is not None:
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
else:
scores_for_choice = scores
if hash_indices_table is not None:
assert input_ids is not None
topk_ids = hash_indices_table[input_ids.long()]
if bias_vl is None or image_sentinel_lo == 0:
topk_ids = hash_indices_table[input_ids.long()]
else:
image_mask = (input_ids.long() >= image_sentinel_lo) & (
input_ids.long() < image_sentinel_lo + 5
)
topk_ids = hash_indices_table[input_ids.long()]
vl_ids = (scores + bias_vl).topk(topk, dim=-1).indices
topk_ids = torch.where(
image_mask.unsqueeze(-1), vl_ids.to(topk_ids.dtype), topk_ids
)
else:
if bias_vl is not None and image_sentinel_lo > 0:
# Image tokens carry five consecutive in-vocab sentinel ids
# starting at image_sentinel_lo and select experts with bias_vl
# instead of the regular bias.
assert input_ids is not None
assert e_score_correction_bias is not None
image_mask = (
(input_ids.long() >= image_sentinel_lo)
& (input_ids.long() < image_sentinel_lo + 5)
).unsqueeze(-1)
row_bias = torch.where(
image_mask,
bias_vl.unsqueeze(0),
e_score_correction_bias.unsqueeze(0),
)
scores_for_choice = scores + row_bias
elif e_score_correction_bias is not None:
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
else:
scores_for_choice = scores
# Match the fused kernel's deterministic tie-break: lower expert ids
# win when scores are equal. torch.topk does not guarantee which tied
# index it selects at the k-th boundary.
@@ -368,3 +396,227 @@ def test_fused_topk_softplus_sqrt_padding(
sorted_w_ref = topk_weights_ref[rows_to_compare].gather(1, idx_ref)
sorted_w = topk_weights[rows_to_compare].gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
def _make_mixed_input_ids(
num_tokens: int, image_sentinel_lo: int, dtype: torch.dtype = torch.long
) -> torch.Tensor:
"""Mix of text, image-sentinel, and above-sentinel ids.
Every third token is an image sentinel id (cycling through the five slots
[image_sentinel_lo, image_sentinel_lo + 5)); every sixth token carries an
id just above the sentinel block, simulating the named special tokens
that follow it — those must route as regular text tokens.
"""
input_ids = torch.randint(
0, image_sentinel_lo, (num_tokens,), dtype=torch.long, device="cuda"
)
pos = torch.arange(num_tokens, device="cuda")
image_rows = pos % 3 == 0
input_ids[image_rows] = image_sentinel_lo + (pos[image_rows] // 3) % 5
above_rows = pos % 6 == 3
input_ids[above_rows] = image_sentinel_lo + 5
return input_ids.to(dtype)
def _assert_topk_matches(
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights_ref: torch.Tensor,
topk_ids_ref: torch.Tensor,
) -> None:
# Different kernels may return the topk experts in different orders when
# scores tie; sort by expert id before comparing.
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
torch.testing.assert_close(
sorted_ref_ids, sorted_ids.to(sorted_ref_ids.dtype), atol=0, rtol=0
)
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops.to(torch.long))
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize(
("num_experts", "topk", "renormalize", "dtype"),
[
(256, 6, True, torch.float32), # Triton dsv4 fast path
(256, 8, True, torch.bfloat16), # CUDA topk kernel
(384, 6, False, torch.half),
(512, 16, True, torch.float32),
],
)
def test_fused_topk_softplus_sqrt_bias_vl(
num_experts: int,
topk: int,
renormalize: bool,
dtype: torch.dtype,
):
"""Image sentinel tokens must select experts with bias_vl.
Rows with ids just above the sentinel block (image_sentinel_lo + 5)
simulate the named special tokens that follow it and must route through
the regular bias path; the reference comparison covers them.
"""
torch.manual_seed(0)
num_tokens = 64
hidden_size = 1024
vocab_size = 128
image_sentinel_lo = vocab_size - 8
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
e_score_correction_bias = torch.randn(
(num_experts,), dtype=torch.float32, device="cuda"
)
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
assert (input_ids == image_sentinel_lo + 5).any()
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=1.5,
e_score_correction_bias=e_score_correction_bias,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=e_score_correction_bias,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=1.5,
input_tokens=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize(
("num_experts", "topk", "renormalize", "dtype", "indices_dtype"),
[
(256, 6, True, torch.float32, torch.long), # specialized hash kernel
(384, 6, True, torch.float32, torch.int32), # specialized hash kernel
(384, 8, False, torch.bfloat16, torch.long), # generic hash kernel
(512, 6, True, torch.float32, torch.long), # generic hash kernel
],
)
def test_fused_topk_softplus_sqrt_hash_bias_vl(
num_experts: int,
topk: int,
renormalize: bool,
dtype: torch.dtype,
indices_dtype: torch.dtype,
):
"""Hash MoE: text rows use tid2eid, image rows use topk(score + bias_vl).
Sentinel ids are in-vocab; rows with ids above the sentinel block
(image_sentinel_lo + 5) must still come straight from the hash table.
"""
torch.manual_seed(0)
num_tokens = 64
hidden_size = 1024
vocab_size = 64
image_sentinel_lo = vocab_size - 8
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
hash_indices_table = torch.stack(
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
).to(device="cuda", dtype=indices_dtype)
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
image_rows = (input_ids >= image_sentinel_lo) & (input_ids < image_sentinel_lo + 5)
assert image_rows.any() and (~image_rows).any()
assert (input_ids == image_sentinel_lo + 5).any()
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=2.5,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=None,
topk=topk,
renormalize=renormalize,
input_tokens=input_ids,
hash_indices_table=hash_indices_table,
routed_scaling_factor=2.5,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
# Text rows must come straight from the hash table, image rows from
# topk(score + bias_vl).
text_ids = hash_indices_table[input_ids[~image_rows].long()]
torch.testing.assert_close(
topk_ids[~image_rows].to(text_ids.dtype), text_ids, atol=0, rtol=0
)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="The DeepSeek V4 fast path is CUDA-only.",
)
def test_dsv4_fast_topk_bias_vl():
torch.manual_seed(0)
num_tokens = 33
num_experts = 256
vocab_size = 128
image_sentinel_lo = vocab_size - 8
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
)
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
bias_vl = torch.randn(num_experts, dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
topk_weights, topk_ids = dsv4_topk(
gating_output,
correction_bias,
torch.int64,
1.5,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
assert topk_ids.dtype == torch.int64
torch.testing.assert_close(topk_ids_ref.to(torch.int64), topk_ids, atol=0, rtol=0)
torch.testing.assert_close(topk_weights_ref, topk_weights, atol=2e-5, rtol=2e-5)
@@ -164,6 +164,11 @@ def test_model_tensor_schema(model_id: str):
"Kimi-K2.5's offline inference has issues about vision chunks. Fix later."
)
if model_id == "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp" and not (
current_platform.is_cuda()
):
pytest.skip("Deepseek V4 is only supported on CUDA")
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
+3
View File
@@ -833,6 +833,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"DeepseekOCR2ForCausalLM": _HfExamplesInfo(
"deepseek-ai/DeepSeek-OCR-2",
),
"DeepseekV4ForConditionalGeneration": _HfExamplesInfo(
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
),
"Dots3NoteForCausalLM": _HfExamplesInfo(
"dots-studio/dots3-note-prev",
is_available_online=False,
+6
View File
@@ -132,6 +132,12 @@ def can_initialize(
f"capability {capability.major}.{capability.minor}"
)
if model_arch == "DeepseekV4ForConditionalGeneration":
from vllm.platforms import current_platform
if not current_platform.is_cuda():
pytest.skip("Deepseek V4 is only supported on CUDA")
with (
patch.object(V1EngineCore, "_initialize_kv_caches", _initialize_kv_caches_v1),
monkeypatch.context() as m,
+5
View File
@@ -64,6 +64,11 @@ def test_registry_imports(model_arch):
):
pytest.skip("HY V4 is only supported on CUDA")
if model_arch == "DeepseekV4ForConditionalGeneration" and (
current_platform.is_cuda()
):
pytest.skip("Deepseek V4 is only supported on CUDA")
# Ensure all model classes can be imported successfully
model_cls = ModelRegistry._try_load_model_cls(model_arch)
assert model_cls is not None
+58
View File
@@ -6,6 +6,7 @@ from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
from vllm.entrypoints.chat_utils import parse_chat_messages
from vllm.renderers.registry import RENDERER_REGISTRY
@@ -427,3 +428,60 @@ def test_deepseek_v4_encode_messages_rejects_invalid_arguments(kwargs):
with pytest.raises(ValueError):
encode_messages([{"role": "user", "content": "Hello"}], **kwargs)
def test_deepseek_v4_image_blocks_become_placeholders():
prompt = _tokenizer().apply_chat_template(
[
{
"role": "user",
"content": [
{"type": "text", "text": "first:"},
{"type": "image_url", "image_url": {"url": "file:///a.png"}},
{"type": "text", "text": "second:"},
{"type": "image", "source": {"data": "AAAA"}},
],
}
],
tokenize=False,
thinking=False,
)
assert "<|User|>first:<|deepseek_image|>second:<|deepseek_image|>" in prompt
def test_deepseek_v4_image_sentinel_ids_match_tokenizer():
"""The borrowed sentinel ids must line up with the reserved
``<|place_holder_mm_span_XXXX|>`` tokens in the tokenizer."""
from vllm.models.deepseek_v4.common.mm_preprocess import (
IMAGE_SENTINEL_BASE_ID,
IMAGE_SENTINEL_TOKEN_NAMES,
image_sentinel_mask,
validate_image_sentinel_ids,
)
class FakeTokenizer:
def __init__(self, offset: int = 0) -> None:
self.offset = offset
def convert_tokens_to_ids(self, token: str) -> int:
return (
IMAGE_SENTINEL_BASE_ID
+ self.offset
+ (IMAGE_SENTINEL_TOKEN_NAMES.index(token))
)
validate_image_sentinel_ids(FakeTokenizer()) # no raise
with pytest.raises(ValueError, match="sentinel"):
validate_image_sentinel_ids(FakeTokenizer(offset=1))
ids = torch.tensor(
[
1,
IMAGE_SENTINEL_BASE_ID,
IMAGE_SENTINEL_BASE_ID + 4,
IMAGE_SENTINEL_BASE_ID + 5,
129264,
]
)
assert image_sentinel_mask(ids).tolist() == [False, True, True, False, False]
@@ -0,0 +1,580 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for DeepSeek-V4 in-image bidirectional SWA visibility (vision variant).
The reference semantics are a torch port of the official
``get_image_visible`` / ``get_window_topk_idxs_visible``: inside an image span
[span_start, span_end] (inclusive), a token at ``pos`` sees
``min(pos - span_start, max_image_tokens - 1)`` extra tokens to the left and
``min(span_end - pos, max_image_tokens)`` to the right; the window then starts
at ``max(pos - (window - 1) - max(left - (window - 1), 0), 0)`` and ends at
``pos + right`` (inclusive).
"""
import pytest
import torch
from typing_extensions import TypedDict
from tests.v1.attention.utils import create_vllm_config
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
combine_topk_swa_indices,
)
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.sparse_swa import (
DeepseekSparseSWAMetadataBuilder,
_compute_image_visibility_kernel,
_compute_swa_indices_and_lens_kernel,
)
from vllm.v1.kv_cache_interface import SlidingWindowMLASpec
WINDOW = 8
MAX_IMG = 6
WIDTH = WINDOW + MAX_IMG
BLOCK_SIZE = 64
def ref_left_right(
seq_lens: list[int],
query_lens: list[int],
spans_per_req: list[list[tuple[int, int]]],
max_image_tokens: int,
) -> tuple[list[int], list[int]]:
"""Per-token (left, right) for the flattened decode-first token stream."""
lefts: list[int] = []
rights: list[int] = []
for seq_len, query_len, spans in zip(seq_lens, query_lens, spans_per_req):
prefix_len = seq_len - query_len
for i in range(query_len):
pos = prefix_len + i
left = right = 0
for span_start, span_end in spans:
if span_start <= pos <= span_end:
left = min(pos - span_start, max_image_tokens - 1)
right = min(span_end - pos, max_image_tokens)
lefts.append(left)
rights.append(right)
return lefts, rights
def ref_swa_bounds(pos: int, window: int, left: int, right: int) -> tuple[int, int]:
"""Reference [start, end) window bounds for one query token."""
left_add = max(left - (window - 1), 0)
start = max(pos - (window - 1) - left_add, 0)
return start, pos + right + 1
def make_batch(
seq_lens: list[int],
query_lens: list[int],
device: torch.device,
):
"""Build the kernel inputs for a decode-first batch."""
num_reqs = len(seq_lens)
query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
query_start_loc[1:] = torch.tensor(
query_lens, dtype=torch.int32, device=device
).cumsum(0)
num_tokens = int(query_start_loc[-1])
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device)
token_to_req = torch.repeat_interleave(
torch.arange(num_reqs, dtype=torch.int32, device=device),
torch.tensor(query_lens, dtype=torch.int32, device=device),
)
max_blocks = (max(seq_lens) + BLOCK_SIZE - 1) // BLOCK_SIZE
block_table = torch.arange(
num_reqs * max_blocks, dtype=torch.int32, device=device
).view(num_reqs, max_blocks)
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
return query_start_loc, seq_lens_t, token_to_req, slot_mapping, block_table
def ref_swa_slot_rows(
seq_lens: list[int],
query_lens: list[int],
spans_per_req: list[list[tuple[int, int]]],
block_table: torch.Tensor,
window: int,
max_image_tokens: int,
width: int,
) -> tuple[list[list[int]], list[int]]:
"""Reference paged slot-id rows and lens for every token in the batch."""
block_table_cpu = block_table.cpu()
lefts, rights = ref_left_right(
seq_lens, query_lens, spans_per_req, max_image_tokens
)
rows: list[list[int]] = []
lens: list[int] = []
token = 0
for req, (seq_len, query_len) in enumerate(zip(seq_lens, query_lens)):
prefix_len = seq_len - query_len
for i in range(query_len):
pos = prefix_len + i
start, end = ref_swa_bounds(pos, window, lefts[token], rights[token])
row = []
for p in range(start, end):
blk = int(block_table_cpu[req, p // BLOCK_SIZE])
row.append(blk * BLOCK_SIZE + p % BLOCK_SIZE)
lens.append(len(row))
row.extend([-1] * (width - len(row)))
rows.append(row)
token += 1
return rows, lens
def run_swa_kernel(
seq_lens: list[int],
query_lens: list[int],
spans_per_req: list[list[tuple[int, int]]],
window: int = WINDOW,
max_image_tokens: int = MAX_IMG,
with_image: bool = True,
) -> tuple[torch.Tensor, torch.Tensor]:
device = torch.device("cuda")
query_start_loc, seq_lens_t, token_to_req, slot_mapping, block_table = make_batch(
seq_lens, query_lens, device
)
num_tokens = int(query_start_loc[-1])
width = window + (max_image_tokens if with_image else 0)
swa_indices = torch.zeros(num_tokens, 1, width, dtype=torch.int32, device=device)
swa_lens = torch.zeros(num_tokens, dtype=torch.int32, device=device)
is_valid = slot_mapping >= 0
if with_image:
lefts, rights = ref_left_right(
seq_lens, query_lens, spans_per_req, max_image_tokens
)
left_t = torch.tensor(lefts, dtype=torch.int32, device=device)
right_t = torch.tensor(rights, dtype=torch.int32, device=device)
else:
left_t = right_t = swa_lens # unused dummies
_compute_swa_indices_and_lens_kernel[(num_tokens,)](
swa_indices,
swa_indices.stride(0),
swa_lens,
window,
width,
left_t,
right_t,
query_start_loc,
seq_lens_t,
token_to_req,
is_valid,
block_table,
block_table.stride(0),
BLOCK_SIZE,
token_offset=0,
HAS_IMAGE=with_image,
TRITON_BLOCK_SIZE=1024,
)
return swa_indices[:, 0], swa_lens
# seq_lens, query_lens, spans per request (positions are prompt-absolute,
# inclusive on both ends, matching the reference's sentinel-bracketed spans).
class _Case(TypedDict):
seq_lens: list[int]
query_lens: list[int]
spans: list[list[tuple[int, int]]]
CASES: list[_Case] = [
# two image spans in one request, one pure-text request
{
"seq_lens": [30, 12],
"query_lens": [30, 12],
"spans": [[(4, 12), (20, 24)], []],
},
# span larger than max_image_tokens, straddling the window boundary
{
"seq_lens": [26],
"query_lens": [26],
"spans": [[(2, 17)]],
},
# chunked prefill: span fully inside the second chunk's query
{
"seq_lens": [40, 9],
"query_lens": [16, 9],
"spans": [[(30, 38)], []],
},
# span ending exactly at the prompt end; tiny request
{
"seq_lens": [7, 15],
"query_lens": [7, 15],
"spans": [[(0, 6)], [(10, 14)]],
},
]
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("case", CASES)
def test_swa_indices_kernel_with_image_spans(case):
rows, lens = ref_swa_slot_rows(
case["seq_lens"],
case["query_lens"],
case["spans"],
make_batch(case["seq_lens"], case["query_lens"], torch.device("cuda"))[4],
WINDOW,
MAX_IMG,
WIDTH,
)
indices, actual_lens = run_swa_kernel(
case["seq_lens"], case["query_lens"], case["spans"], with_image=True
)
assert actual_lens.cpu().tolist() == lens
assert indices.cpu().tolist() == rows
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("case", CASES)
def test_swa_indices_kernel_without_image_unchanged(case):
"""HAS_IMAGE=False must reproduce the plain causal sliding window."""
rows, lens = ref_swa_slot_rows(
case["seq_lens"],
case["query_lens"],
[[] for _ in case["seq_lens"]],
make_batch(case["seq_lens"], case["query_lens"], torch.device("cuda"))[4],
WINDOW,
MAX_IMG,
WINDOW,
)
indices, actual_lens = run_swa_kernel(
case["seq_lens"], case["query_lens"], case["spans"], with_image=False
)
assert indices.shape[-1] == WINDOW
assert actual_lens.cpu().tolist() == lens
assert indices.cpu().tolist() == rows
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_image_visibility_kernel():
"""The builder's visibility kernel must match get_image_visible."""
device = torch.device("cuda")
seq_lens = [30, 12]
query_lens = [30, 12]
spans = [[(4, 12), (20, 24)], []]
query_start_loc, seq_lens_t, token_to_req, _, _ = make_batch(
seq_lens, query_lens, device
)
num_tokens = int(query_start_loc[-1])
# CSR span layout: request -> contiguous [start, end) span list.
indptr = [0, 2, 2]
starts = [4, 20]
ends = [12, 24]
left = torch.zeros(num_tokens, dtype=torch.int32, device=device)
right = torch.zeros(num_tokens, dtype=torch.int32, device=device)
_compute_image_visibility_kernel[(num_tokens,)](
left,
right,
torch.tensor(indptr, dtype=torch.int32, device=device),
torch.tensor(starts, dtype=torch.int32, device=device),
torch.tensor(ends, dtype=torch.int32, device=device),
query_start_loc,
seq_lens_t,
token_to_req,
MAX_IMG,
token_offset=0,
)
ref_left, ref_right = ref_left_right(seq_lens, query_lens, spans, MAX_IMG)
assert left.cpu().tolist() == ref_left
assert right.cpu().tolist() == ref_right
def combine_case(
compress_ratio: int,
topk: int,
seq_lens: list[int],
query_lens: list[int],
spans: list[list[tuple[int, int]]],
with_image: bool,
):
"""Run combine_topk_swa_indices and return (indices, lens, expected)."""
device = torch.device("cuda")
num_reqs = len(seq_lens)
query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
query_start_loc[1:] = torch.tensor(
query_lens, dtype=torch.int32, device=device
).cumsum(0)
num_tokens = int(query_start_loc[-1])
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device)
gather_lens = torch.tensor(
[q + min(s - q, WINDOW - 1) for s, q in zip(seq_lens, query_lens)],
dtype=torch.int32,
device=device,
)
N = (max(seq_lens) + compress_ratio - 1) // compress_ratio
M = N + int(gather_lens.max()) + 8
gen = torch.Generator(device="cpu").manual_seed(0)
topk_indices = torch.randint(
0, 4096, (num_tokens, max(topk, 1)), generator=gen, dtype=torch.int32
).to(device)
topk_indices = topk_indices[:, : max(topk, 1)]
if with_image:
lefts, rights = ref_left_right(seq_lens, query_lens, spans, MAX_IMG)
left_t = torch.tensor(lefts, dtype=torch.int32, device=device)
right_t = torch.tensor(rights, dtype=torch.int32, device=device)
else:
left_t = right_t = None
combined_indices, combined_lens = combine_topk_swa_indices(
topk_indices,
query_start_loc,
seq_lens_t,
gather_lens,
WINDOW,
compress_ratio,
topk,
M,
N,
left_visible=left_t,
right_visible=right_t,
max_image_tokens=MAX_IMG,
)
# Reference rows.
lefts, rights = ref_left_right(
seq_lens,
query_lens,
spans if with_image else [[] for _ in seq_lens],
MAX_IMG,
)
topk_cpu = topk_indices.cpu()
width = WINDOW + MAX_IMG
combined_topk = (topk + width + 127) // 128 * 128
rows = []
lens = []
token = 0
for b, (seq_len, query_len) in enumerate(zip(seq_lens, query_lens)):
prefix_len = seq_len - query_len
gather_start = seq_len - int(gather_lens[b])
for i in range(query_len):
pos = prefix_len + i
topk_len = min((pos + 1) // compress_ratio, topk)
start, end = ref_swa_bounds(pos, WINDOW, lefts[token], rights[token])
swa_len = end - start
row = [-1] * combined_topk
for j in range(topk_len):
row[j] = int(topk_cpu[token, j]) + M * b
for j in range(swa_len):
row[topk_len + j] = M * b + N + start + j - gather_start
rows.append(row)
lens.append(topk_len + swa_len)
token += 1
return combined_indices, combined_lens, rows, lens
COMBINE_CASES = [
dict(compress_ratio=1, topk=0), # SWA-only layer
dict(compress_ratio=4, topk=16), # C4A layer
]
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("cfg", COMBINE_CASES)
def test_combine_topk_swa_with_image_spans(cfg):
case = CASES[0]
indices, lens, rows, exp_lens = combine_case(
cfg["compress_ratio"],
cfg["topk"],
case["seq_lens"],
case["query_lens"],
case["spans"],
with_image=True,
)
assert lens.cpu().tolist() == exp_lens
assert indices.cpu().tolist() == rows
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("cfg", COMBINE_CASES)
def test_combine_topk_swa_without_image_unchanged(cfg):
"""left_visible=None must reproduce the plain causal combined indices."""
case = CASES[0]
indices, lens, rows, exp_lens = combine_case(
cfg["compress_ratio"],
cfg["topk"],
case["seq_lens"],
case["query_lens"],
case["spans"],
with_image=False,
)
assert lens.cpu().tolist() == exp_lens
assert indices.cpu().tolist() == rows
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_flashinfer_mixed_sparse_indices_with_image_spans():
device = torch.device("cuda")
# 1 decode token (req 0) + two prefill requests (reqs 1, 2).
seq_lens = [20, 30, 9]
query_lens = [1, 30, 9]
spans = [[], [(4, 12)], [(0, 6)]]
query_start_loc = torch.tensor([0, 1, 31, 40], dtype=torch.int32, device=device)
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device)
token_to_req = torch.tensor(
[0] + [1] * 30 + [2] * 9, dtype=torch.int32, device=device
)
block_table = torch.arange(3, dtype=torch.int32, device=device).view(3, 1)
decode_swa = torch.tensor(
[[100, 101, 102, 103, 104, 105, 106, 107]], dtype=torch.int32, device=device
)
prefill_topk = torch.zeros(39, 0, dtype=torch.int32, device=device)
lefts, rights = ref_left_right(seq_lens, query_lens, spans, MAX_IMG)
left_t = torch.tensor(lefts, dtype=torch.int32, device=device)
right_t = torch.tensor(rights, dtype=torch.int32, device=device)
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=decode_swa,
decode_compressed_indices=None,
decode_compressed_topk_lens=None,
prefill_topk_indices=prefill_topk,
query_start_loc=query_start_loc,
seq_lens=seq_lens_t,
token_to_req_indices=token_to_req,
swa_block_table=block_table,
swa_block_size=BLOCK_SIZE,
compressed_block_table=None,
compressed_block_size=BLOCK_SIZE,
window_size=WINDOW,
compress_ratio=1,
topk=0,
prefill_left_visible=left_t,
prefill_right_visible=right_t,
max_image_tokens=MAX_IMG,
)
swa_total = WINDOW + MAX_IMG
assert sparse_indices.shape == (40, swa_total)
assert sparse_lens.cpu().tolist() == [swa_total] * 40
# Decode row: slots copied, image-extension columns padded with -1.
decode_row = sparse_indices[0].cpu().tolist()
assert decode_row[:WINDOW] == list(range(100, 108))
assert decode_row[WINDOW:] == [-1] * MAX_IMG
# Prefill rows: paged slot ids over the widened window.
exp_rows, _ = ref_swa_slot_rows(
seq_lens, query_lens, spans, block_table, WINDOW, MAX_IMG, swa_total
)
actual = sparse_indices[1:].cpu().tolist()
assert actual == exp_rows[1:]
def make_builder(vision: bool) -> DeepseekSparseSWAMetadataBuilder:
overrides: dict = {"sliding_window": WINDOW}
if vision:
overrides.update(vision_n_layers=2, vision_max_n_token=MAX_IMG)
vllm_config = create_vllm_config(
max_model_len=4096,
max_num_batched_tokens=64,
max_num_seqs=8,
hf_config_override=overrides,
)
spec = SlidingWindowMLASpec(
block_size=BLOCK_SIZE,
num_kv_heads=1,
head_size=512,
dtype=torch.bfloat16,
sliding_window=WINDOW,
cache_dtype_str="auto",
model_version="deepseek_v4",
)
return DeepseekSparseSWAMetadataBuilder(
kv_cache_spec=spec,
layer_names=["layer0"],
vllm_config=vllm_config,
device=torch.device("cuda"),
)
def build_metadata(
builder: DeepseekSparseSWAMetadataBuilder,
seq_lens: list[int],
query_lens: list[int],
mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None,
):
device = torch.device("cuda")
query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch(
seq_lens, query_lens, device
)
return builder.build(
0,
CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=seq_lens_t,
seq_lens_cpu_upper_bound=seq_lens_t.cpu(),
num_reqs=len(seq_lens),
num_actual_tokens=int(query_start_loc[-1]),
max_query_len=max(query_lens),
max_seq_len=max(seq_lens),
block_table_tensor=block_table,
slot_mapping=slot_mapping,
causal=True,
mm_req_doc_ranges=mm_req_doc_ranges,
),
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_builder_in_image_visibility():
seq_lens = [30, 12]
query_lens = [30, 12]
spans = [[(4, 12), (20, 24)], []]
builder = make_builder(vision=True)
md = build_metadata(builder, seq_lens, query_lens, {0: spans[0], 1: spans[1]})
assert md.num_prefills == 2
assert md.num_decode_tokens == 0
assert md.prefill_swa_indices.shape[-1] == WIDTH
assert md.prefill_left_visible is not None
assert md.prefill_right_visible is not None
ref_left, ref_right = ref_left_right(seq_lens, query_lens, spans, MAX_IMG)
assert md.prefill_left_visible.cpu().tolist() == ref_left
assert md.prefill_right_visible.cpu().tolist() == ref_right
_, _, _, _, block_table = make_batch(seq_lens, query_lens, torch.device("cuda"))
rows, lens = ref_swa_slot_rows(
seq_lens, query_lens, spans, block_table, WINDOW, MAX_IMG, WIDTH
)
assert md.prefill_swa_lens.cpu().tolist() == lens
assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_builder_no_image_spans_fast_path():
"""Vision model, image-free batch: no visibility tensors, plain window."""
seq_lens = [30, 12]
query_lens = [30, 12]
builder = make_builder(vision=True)
md = build_metadata(builder, seq_lens, query_lens, {0: [], 1: []})
assert md.prefill_left_visible is None
assert md.prefill_right_visible is None
_, _, _, _, block_table = make_batch(seq_lens, query_lens, torch.device("cuda"))
rows, lens = ref_swa_slot_rows(
seq_lens, query_lens, [[], []], block_table, WINDOW, MAX_IMG, WIDTH
)
assert md.prefill_swa_lens.cpu().tolist() == lens
assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_builder_text_model_unchanged():
"""Text-only model: buffers stay window-sized and spans are ignored."""
seq_lens = [30, 12]
query_lens = [30, 12]
builder = make_builder(vision=False)
assert builder.max_image_tokens == 0
md = build_metadata(builder, seq_lens, query_lens, None)
assert md.prefill_swa_indices.shape[-1] == WINDOW
assert md.prefill_left_visible is None
_, _, _, _, block_table = make_batch(seq_lens, query_lens, torch.device("cuda"))
rows, lens = ref_swa_slot_rows(
seq_lens, query_lens, [[], []], block_table, WINDOW, MAX_IMG, WINDOW
)
assert md.prefill_swa_lens.cpu().tolist() == lens
assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows