[New model][Multimodal] Add DeepSeek-V4-Flash-Vision-Exp support (#54566)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -465,3 +465,34 @@ def test_draft_model_arch_config(
|
||||
_assert_model_config_methods(
|
||||
model_config, expected, check_head_size=check_head_size
|
||||
)
|
||||
|
||||
|
||||
def test_deepseek_v4_convertor_splits_vision_architecture():
|
||||
"""The vision checkpoint shares model_type/architectures with the
|
||||
text-only DeepSeek-V4; the convertor routes it to the VL wrapper class
|
||||
by rewriting hf_config.architectures (model-class resolution reads the
|
||||
raw attribute)."""
|
||||
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
DeepseekV4ModelArchConfigConvertor,
|
||||
)
|
||||
|
||||
text_cfg = DeepseekV4Config(architectures=["DeepseekV4ForCausalLM"])
|
||||
conv = DeepseekV4ModelArchConfigConvertor(text_cfg, text_cfg)
|
||||
assert conv.get_architectures() == ["DeepseekV4ForCausalLM"]
|
||||
assert text_cfg.architectures == ["DeepseekV4ForCausalLM"]
|
||||
|
||||
vl_cfg = DeepseekV4Config(
|
||||
architectures=["DeepseekV4ForCausalLM"], vision_n_layers=32
|
||||
)
|
||||
conv = DeepseekV4ModelArchConfigConvertor(vl_cfg, vl_cfg)
|
||||
assert conv.get_architectures() == ["DeepseekV4ForConditionalGeneration"]
|
||||
assert vl_cfg.architectures == ["DeepseekV4ForConditionalGeneration"]
|
||||
|
||||
# Speculative-draft configs (set by SpeculativeConfig.hf_config_override)
|
||||
# keep their own architecture even with a vision tower in the config.
|
||||
for draft_arch in ("DSparkDraftModel", "DeepSeekV4MTPModel"):
|
||||
draft_cfg = DeepseekV4Config(architectures=[draft_arch], vision_n_layers=32)
|
||||
conv = DeepseekV4ModelArchConfigConvertor(draft_cfg, draft_cfg)
|
||||
assert conv.get_architectures() == [draft_arch]
|
||||
assert draft_cfg.architectures == [draft_arch]
|
||||
|
||||
@@ -26,18 +26,46 @@ def _torch_topk_softplus_sqrt(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
):
|
||||
scores = F.softplus(gating_output.float()).sqrt()
|
||||
original_scores = scores
|
||||
if e_score_correction_bias is not None:
|
||||
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
|
||||
else:
|
||||
scores_for_choice = scores
|
||||
|
||||
if hash_indices_table is not None:
|
||||
assert input_ids is not None
|
||||
topk_ids = hash_indices_table[input_ids.long()]
|
||||
if bias_vl is None or image_sentinel_lo == 0:
|
||||
topk_ids = hash_indices_table[input_ids.long()]
|
||||
else:
|
||||
image_mask = (input_ids.long() >= image_sentinel_lo) & (
|
||||
input_ids.long() < image_sentinel_lo + 5
|
||||
)
|
||||
topk_ids = hash_indices_table[input_ids.long()]
|
||||
vl_ids = (scores + bias_vl).topk(topk, dim=-1).indices
|
||||
topk_ids = torch.where(
|
||||
image_mask.unsqueeze(-1), vl_ids.to(topk_ids.dtype), topk_ids
|
||||
)
|
||||
else:
|
||||
if bias_vl is not None and image_sentinel_lo > 0:
|
||||
# Image tokens carry five consecutive in-vocab sentinel ids
|
||||
# starting at image_sentinel_lo and select experts with bias_vl
|
||||
# instead of the regular bias.
|
||||
assert input_ids is not None
|
||||
assert e_score_correction_bias is not None
|
||||
image_mask = (
|
||||
(input_ids.long() >= image_sentinel_lo)
|
||||
& (input_ids.long() < image_sentinel_lo + 5)
|
||||
).unsqueeze(-1)
|
||||
row_bias = torch.where(
|
||||
image_mask,
|
||||
bias_vl.unsqueeze(0),
|
||||
e_score_correction_bias.unsqueeze(0),
|
||||
)
|
||||
scores_for_choice = scores + row_bias
|
||||
elif e_score_correction_bias is not None:
|
||||
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
|
||||
else:
|
||||
scores_for_choice = scores
|
||||
# Match the fused kernel's deterministic tie-break: lower expert ids
|
||||
# win when scores are equal. torch.topk does not guarantee which tied
|
||||
# index it selects at the k-th boundary.
|
||||
@@ -368,3 +396,227 @@ def test_fused_topk_softplus_sqrt_padding(
|
||||
sorted_w_ref = topk_weights_ref[rows_to_compare].gather(1, idx_ref)
|
||||
sorted_w = topk_weights[rows_to_compare].gather(1, idx_ops)
|
||||
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def _make_mixed_input_ids(
|
||||
num_tokens: int, image_sentinel_lo: int, dtype: torch.dtype = torch.long
|
||||
) -> torch.Tensor:
|
||||
"""Mix of text, image-sentinel, and above-sentinel ids.
|
||||
|
||||
Every third token is an image sentinel id (cycling through the five slots
|
||||
[image_sentinel_lo, image_sentinel_lo + 5)); every sixth token carries an
|
||||
id just above the sentinel block, simulating the named special tokens
|
||||
that follow it — those must route as regular text tokens.
|
||||
"""
|
||||
input_ids = torch.randint(
|
||||
0, image_sentinel_lo, (num_tokens,), dtype=torch.long, device="cuda"
|
||||
)
|
||||
pos = torch.arange(num_tokens, device="cuda")
|
||||
image_rows = pos % 3 == 0
|
||||
input_ids[image_rows] = image_sentinel_lo + (pos[image_rows] // 3) % 5
|
||||
above_rows = pos % 6 == 3
|
||||
input_ids[above_rows] = image_sentinel_lo + 5
|
||||
return input_ids.to(dtype)
|
||||
|
||||
|
||||
def _assert_topk_matches(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights_ref: torch.Tensor,
|
||||
topk_ids_ref: torch.Tensor,
|
||||
) -> None:
|
||||
# Different kernels may return the topk experts in different orders when
|
||||
# scores tie; sort by expert id before comparing.
|
||||
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
|
||||
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
|
||||
torch.testing.assert_close(
|
||||
sorted_ref_ids, sorted_ids.to(sorted_ref_ids.dtype), atol=0, rtol=0
|
||||
)
|
||||
|
||||
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
|
||||
sorted_w = topk_weights.gather(1, idx_ops.to(torch.long))
|
||||
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "topk", "renormalize", "dtype"),
|
||||
[
|
||||
(256, 6, True, torch.float32), # Triton dsv4 fast path
|
||||
(256, 8, True, torch.bfloat16), # CUDA topk kernel
|
||||
(384, 6, False, torch.half),
|
||||
(512, 16, True, torch.float32),
|
||||
],
|
||||
)
|
||||
def test_fused_topk_softplus_sqrt_bias_vl(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
"""Image sentinel tokens must select experts with bias_vl.
|
||||
|
||||
Rows with ids just above the sentinel block (image_sentinel_lo + 5)
|
||||
simulate the named special tokens that follow it and must route through
|
||||
the regular bias path; the reference comparison covers them.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 64
|
||||
hidden_size = 1024
|
||||
vocab_size = 128
|
||||
image_sentinel_lo = vocab_size - 8
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
e_score_correction_bias = torch.randn(
|
||||
(num_experts,), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
|
||||
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
|
||||
assert (input_ids == image_sentinel_lo + 5).any()
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
input_ids=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=1.5,
|
||||
input_tokens=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="This test is skipped on non-CUDA platform.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "topk", "renormalize", "dtype", "indices_dtype"),
|
||||
[
|
||||
(256, 6, True, torch.float32, torch.long), # specialized hash kernel
|
||||
(384, 6, True, torch.float32, torch.int32), # specialized hash kernel
|
||||
(384, 8, False, torch.bfloat16, torch.long), # generic hash kernel
|
||||
(512, 6, True, torch.float32, torch.long), # generic hash kernel
|
||||
],
|
||||
)
|
||||
def test_fused_topk_softplus_sqrt_hash_bias_vl(
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
dtype: torch.dtype,
|
||||
indices_dtype: torch.dtype,
|
||||
):
|
||||
"""Hash MoE: text rows use tid2eid, image rows use topk(score + bias_vl).
|
||||
|
||||
Sentinel ids are in-vocab; rows with ids above the sentinel block
|
||||
(image_sentinel_lo + 5) must still come straight from the hash table.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 64
|
||||
hidden_size = 1024
|
||||
vocab_size = 64
|
||||
image_sentinel_lo = vocab_size - 8
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
hash_indices_table = torch.stack(
|
||||
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
|
||||
).to(device="cuda", dtype=indices_dtype)
|
||||
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
|
||||
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
|
||||
image_rows = (input_ids >= image_sentinel_lo) & (input_ids < image_sentinel_lo + 5)
|
||||
assert image_rows.any() and (~image_rows).any()
|
||||
assert (input_ids == image_sentinel_lo + 5).any()
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=2.5,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=None,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
input_tokens=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
routed_scaling_factor=2.5,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)
|
||||
|
||||
# Text rows must come straight from the hash table, image rows from
|
||||
# topk(score + bias_vl).
|
||||
text_ids = hash_indices_table[input_ids[~image_rows].long()]
|
||||
torch.testing.assert_close(
|
||||
topk_ids[~image_rows].to(text_ids.dtype), text_ids, atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="The DeepSeek V4 fast path is CUDA-only.",
|
||||
)
|
||||
def test_dsv4_fast_topk_bias_vl():
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 33
|
||||
num_experts = 256
|
||||
vocab_size = 128
|
||||
image_sentinel_lo = vocab_size - 8
|
||||
gating_output = torch.randn(
|
||||
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
|
||||
bias_vl = torch.randn(num_experts, dtype=torch.float32, device="cuda")
|
||||
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=6,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=correction_bias,
|
||||
input_ids=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
topk_weights, topk_ids = dsv4_topk(
|
||||
gating_output,
|
||||
correction_bias,
|
||||
torch.int64,
|
||||
1.5,
|
||||
input_ids=input_ids,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
)
|
||||
|
||||
assert topk_ids.dtype == torch.int64
|
||||
torch.testing.assert_close(topk_ids_ref.to(torch.int64), topk_ids, atol=0, rtol=0)
|
||||
torch.testing.assert_close(topk_weights_ref, topk_weights, atol=2e-5, rtol=2e-5)
|
||||
|
||||
@@ -164,6 +164,11 @@ def test_model_tensor_schema(model_id: str):
|
||||
"Kimi-K2.5's offline inference has issues about vision chunks. Fix later."
|
||||
)
|
||||
|
||||
if model_id == "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp" and not (
|
||||
current_platform.is_cuda()
|
||||
):
|
||||
pytest.skip("Deepseek V4 is only supported on CUDA")
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(
|
||||
|
||||
@@ -833,6 +833,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"DeepseekOCR2ForCausalLM": _HfExamplesInfo(
|
||||
"deepseek-ai/DeepSeek-OCR-2",
|
||||
),
|
||||
"DeepseekV4ForConditionalGeneration": _HfExamplesInfo(
|
||||
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
|
||||
),
|
||||
"Dots3NoteForCausalLM": _HfExamplesInfo(
|
||||
"dots-studio/dots3-note-prev",
|
||||
is_available_online=False,
|
||||
|
||||
@@ -132,6 +132,12 @@ def can_initialize(
|
||||
f"capability {capability.major}.{capability.minor}"
|
||||
)
|
||||
|
||||
if model_arch == "DeepseekV4ForConditionalGeneration":
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Deepseek V4 is only supported on CUDA")
|
||||
|
||||
with (
|
||||
patch.object(V1EngineCore, "_initialize_kv_caches", _initialize_kv_caches_v1),
|
||||
monkeypatch.context() as m,
|
||||
|
||||
@@ -64,6 +64,11 @@ def test_registry_imports(model_arch):
|
||||
):
|
||||
pytest.skip("HY V4 is only supported on CUDA")
|
||||
|
||||
if model_arch == "DeepseekV4ForConditionalGeneration" and (
|
||||
current_platform.is_cuda()
|
||||
):
|
||||
pytest.skip("Deepseek V4 is only supported on CUDA")
|
||||
|
||||
# Ensure all model classes can be imported successfully
|
||||
model_cls = ModelRegistry._try_load_model_cls(model_arch)
|
||||
assert model_cls is not None
|
||||
|
||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.entrypoints.chat_utils import parse_chat_messages
|
||||
from vllm.renderers.registry import RENDERER_REGISTRY
|
||||
@@ -427,3 +428,60 @@ def test_deepseek_v4_encode_messages_rejects_invalid_arguments(kwargs):
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
encode_messages([{"role": "user", "content": "Hello"}], **kwargs)
|
||||
|
||||
|
||||
def test_deepseek_v4_image_blocks_become_placeholders():
|
||||
prompt = _tokenizer().apply_chat_template(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "first:"},
|
||||
{"type": "image_url", "image_url": {"url": "file:///a.png"}},
|
||||
{"type": "text", "text": "second:"},
|
||||
{"type": "image", "source": {"data": "AAAA"}},
|
||||
],
|
||||
}
|
||||
],
|
||||
tokenize=False,
|
||||
thinking=False,
|
||||
)
|
||||
|
||||
assert "<|User|>first:<|deepseek_image|>second:<|deepseek_image|>" in prompt
|
||||
|
||||
|
||||
def test_deepseek_v4_image_sentinel_ids_match_tokenizer():
|
||||
"""The borrowed sentinel ids must line up with the reserved
|
||||
``<|place_holder_mm_span_XXXX|>`` tokens in the tokenizer."""
|
||||
from vllm.models.deepseek_v4.common.mm_preprocess import (
|
||||
IMAGE_SENTINEL_BASE_ID,
|
||||
IMAGE_SENTINEL_TOKEN_NAMES,
|
||||
image_sentinel_mask,
|
||||
validate_image_sentinel_ids,
|
||||
)
|
||||
|
||||
class FakeTokenizer:
|
||||
def __init__(self, offset: int = 0) -> None:
|
||||
self.offset = offset
|
||||
|
||||
def convert_tokens_to_ids(self, token: str) -> int:
|
||||
return (
|
||||
IMAGE_SENTINEL_BASE_ID
|
||||
+ self.offset
|
||||
+ (IMAGE_SENTINEL_TOKEN_NAMES.index(token))
|
||||
)
|
||||
|
||||
validate_image_sentinel_ids(FakeTokenizer()) # no raise
|
||||
with pytest.raises(ValueError, match="sentinel"):
|
||||
validate_image_sentinel_ids(FakeTokenizer(offset=1))
|
||||
|
||||
ids = torch.tensor(
|
||||
[
|
||||
1,
|
||||
IMAGE_SENTINEL_BASE_ID,
|
||||
IMAGE_SENTINEL_BASE_ID + 4,
|
||||
IMAGE_SENTINEL_BASE_ID + 5,
|
||||
129264,
|
||||
]
|
||||
)
|
||||
assert image_sentinel_mask(ids).tolist() == [False, True, True, False, False]
|
||||
|
||||
@@ -0,0 +1,580 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for DeepSeek-V4 in-image bidirectional SWA visibility (vision variant).
|
||||
|
||||
The reference semantics are a torch port of the official
|
||||
``get_image_visible`` / ``get_window_topk_idxs_visible``: inside an image span
|
||||
[span_start, span_end] (inclusive), a token at ``pos`` sees
|
||||
``min(pos - span_start, max_image_tokens - 1)`` extra tokens to the left and
|
||||
``min(span_end - pos, max_image_tokens)`` to the right; the window then starts
|
||||
at ``max(pos - (window - 1) - max(left - (window - 1), 0), 0)`` and ends at
|
||||
``pos + right`` (inclusive).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
combine_topk_swa_indices,
|
||||
)
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
DeepseekSparseSWAMetadataBuilder,
|
||||
_compute_image_visibility_kernel,
|
||||
_compute_swa_indices_and_lens_kernel,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import SlidingWindowMLASpec
|
||||
|
||||
WINDOW = 8
|
||||
MAX_IMG = 6
|
||||
WIDTH = WINDOW + MAX_IMG
|
||||
BLOCK_SIZE = 64
|
||||
|
||||
|
||||
def ref_left_right(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
spans_per_req: list[list[tuple[int, int]]],
|
||||
max_image_tokens: int,
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Per-token (left, right) for the flattened decode-first token stream."""
|
||||
lefts: list[int] = []
|
||||
rights: list[int] = []
|
||||
for seq_len, query_len, spans in zip(seq_lens, query_lens, spans_per_req):
|
||||
prefix_len = seq_len - query_len
|
||||
for i in range(query_len):
|
||||
pos = prefix_len + i
|
||||
left = right = 0
|
||||
for span_start, span_end in spans:
|
||||
if span_start <= pos <= span_end:
|
||||
left = min(pos - span_start, max_image_tokens - 1)
|
||||
right = min(span_end - pos, max_image_tokens)
|
||||
lefts.append(left)
|
||||
rights.append(right)
|
||||
return lefts, rights
|
||||
|
||||
|
||||
def ref_swa_bounds(pos: int, window: int, left: int, right: int) -> tuple[int, int]:
|
||||
"""Reference [start, end) window bounds for one query token."""
|
||||
left_add = max(left - (window - 1), 0)
|
||||
start = max(pos - (window - 1) - left_add, 0)
|
||||
return start, pos + right + 1
|
||||
|
||||
|
||||
def make_batch(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
device: torch.device,
|
||||
):
|
||||
"""Build the kernel inputs for a decode-first batch."""
|
||||
num_reqs = len(seq_lens)
|
||||
query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
query_start_loc[1:] = torch.tensor(
|
||||
query_lens, dtype=torch.int32, device=device
|
||||
).cumsum(0)
|
||||
num_tokens = int(query_start_loc[-1])
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device)
|
||||
token_to_req = torch.repeat_interleave(
|
||||
torch.arange(num_reqs, dtype=torch.int32, device=device),
|
||||
torch.tensor(query_lens, dtype=torch.int32, device=device),
|
||||
)
|
||||
max_blocks = (max(seq_lens) + BLOCK_SIZE - 1) // BLOCK_SIZE
|
||||
block_table = torch.arange(
|
||||
num_reqs * max_blocks, dtype=torch.int32, device=device
|
||||
).view(num_reqs, max_blocks)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
return query_start_loc, seq_lens_t, token_to_req, slot_mapping, block_table
|
||||
|
||||
|
||||
def ref_swa_slot_rows(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
spans_per_req: list[list[tuple[int, int]]],
|
||||
block_table: torch.Tensor,
|
||||
window: int,
|
||||
max_image_tokens: int,
|
||||
width: int,
|
||||
) -> tuple[list[list[int]], list[int]]:
|
||||
"""Reference paged slot-id rows and lens for every token in the batch."""
|
||||
block_table_cpu = block_table.cpu()
|
||||
lefts, rights = ref_left_right(
|
||||
seq_lens, query_lens, spans_per_req, max_image_tokens
|
||||
)
|
||||
rows: list[list[int]] = []
|
||||
lens: list[int] = []
|
||||
token = 0
|
||||
for req, (seq_len, query_len) in enumerate(zip(seq_lens, query_lens)):
|
||||
prefix_len = seq_len - query_len
|
||||
for i in range(query_len):
|
||||
pos = prefix_len + i
|
||||
start, end = ref_swa_bounds(pos, window, lefts[token], rights[token])
|
||||
row = []
|
||||
for p in range(start, end):
|
||||
blk = int(block_table_cpu[req, p // BLOCK_SIZE])
|
||||
row.append(blk * BLOCK_SIZE + p % BLOCK_SIZE)
|
||||
lens.append(len(row))
|
||||
row.extend([-1] * (width - len(row)))
|
||||
rows.append(row)
|
||||
token += 1
|
||||
return rows, lens
|
||||
|
||||
|
||||
def run_swa_kernel(
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
spans_per_req: list[list[tuple[int, int]]],
|
||||
window: int = WINDOW,
|
||||
max_image_tokens: int = MAX_IMG,
|
||||
with_image: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
device = torch.device("cuda")
|
||||
query_start_loc, seq_lens_t, token_to_req, slot_mapping, block_table = make_batch(
|
||||
seq_lens, query_lens, device
|
||||
)
|
||||
num_tokens = int(query_start_loc[-1])
|
||||
width = window + (max_image_tokens if with_image else 0)
|
||||
swa_indices = torch.zeros(num_tokens, 1, width, dtype=torch.int32, device=device)
|
||||
swa_lens = torch.zeros(num_tokens, dtype=torch.int32, device=device)
|
||||
is_valid = slot_mapping >= 0
|
||||
|
||||
if with_image:
|
||||
lefts, rights = ref_left_right(
|
||||
seq_lens, query_lens, spans_per_req, max_image_tokens
|
||||
)
|
||||
left_t = torch.tensor(lefts, dtype=torch.int32, device=device)
|
||||
right_t = torch.tensor(rights, dtype=torch.int32, device=device)
|
||||
else:
|
||||
left_t = right_t = swa_lens # unused dummies
|
||||
|
||||
_compute_swa_indices_and_lens_kernel[(num_tokens,)](
|
||||
swa_indices,
|
||||
swa_indices.stride(0),
|
||||
swa_lens,
|
||||
window,
|
||||
width,
|
||||
left_t,
|
||||
right_t,
|
||||
query_start_loc,
|
||||
seq_lens_t,
|
||||
token_to_req,
|
||||
is_valid,
|
||||
block_table,
|
||||
block_table.stride(0),
|
||||
BLOCK_SIZE,
|
||||
token_offset=0,
|
||||
HAS_IMAGE=with_image,
|
||||
TRITON_BLOCK_SIZE=1024,
|
||||
)
|
||||
return swa_indices[:, 0], swa_lens
|
||||
|
||||
|
||||
# seq_lens, query_lens, spans per request (positions are prompt-absolute,
|
||||
# inclusive on both ends, matching the reference's sentinel-bracketed spans).
|
||||
class _Case(TypedDict):
|
||||
seq_lens: list[int]
|
||||
query_lens: list[int]
|
||||
spans: list[list[tuple[int, int]]]
|
||||
|
||||
|
||||
CASES: list[_Case] = [
|
||||
# two image spans in one request, one pure-text request
|
||||
{
|
||||
"seq_lens": [30, 12],
|
||||
"query_lens": [30, 12],
|
||||
"spans": [[(4, 12), (20, 24)], []],
|
||||
},
|
||||
# span larger than max_image_tokens, straddling the window boundary
|
||||
{
|
||||
"seq_lens": [26],
|
||||
"query_lens": [26],
|
||||
"spans": [[(2, 17)]],
|
||||
},
|
||||
# chunked prefill: span fully inside the second chunk's query
|
||||
{
|
||||
"seq_lens": [40, 9],
|
||||
"query_lens": [16, 9],
|
||||
"spans": [[(30, 38)], []],
|
||||
},
|
||||
# span ending exactly at the prompt end; tiny request
|
||||
{
|
||||
"seq_lens": [7, 15],
|
||||
"query_lens": [7, 15],
|
||||
"spans": [[(0, 6)], [(10, 14)]],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("case", CASES)
|
||||
def test_swa_indices_kernel_with_image_spans(case):
|
||||
rows, lens = ref_swa_slot_rows(
|
||||
case["seq_lens"],
|
||||
case["query_lens"],
|
||||
case["spans"],
|
||||
make_batch(case["seq_lens"], case["query_lens"], torch.device("cuda"))[4],
|
||||
WINDOW,
|
||||
MAX_IMG,
|
||||
WIDTH,
|
||||
)
|
||||
indices, actual_lens = run_swa_kernel(
|
||||
case["seq_lens"], case["query_lens"], case["spans"], with_image=True
|
||||
)
|
||||
assert actual_lens.cpu().tolist() == lens
|
||||
assert indices.cpu().tolist() == rows
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("case", CASES)
|
||||
def test_swa_indices_kernel_without_image_unchanged(case):
|
||||
"""HAS_IMAGE=False must reproduce the plain causal sliding window."""
|
||||
rows, lens = ref_swa_slot_rows(
|
||||
case["seq_lens"],
|
||||
case["query_lens"],
|
||||
[[] for _ in case["seq_lens"]],
|
||||
make_batch(case["seq_lens"], case["query_lens"], torch.device("cuda"))[4],
|
||||
WINDOW,
|
||||
MAX_IMG,
|
||||
WINDOW,
|
||||
)
|
||||
indices, actual_lens = run_swa_kernel(
|
||||
case["seq_lens"], case["query_lens"], case["spans"], with_image=False
|
||||
)
|
||||
assert indices.shape[-1] == WINDOW
|
||||
assert actual_lens.cpu().tolist() == lens
|
||||
assert indices.cpu().tolist() == rows
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_image_visibility_kernel():
|
||||
"""The builder's visibility kernel must match get_image_visible."""
|
||||
device = torch.device("cuda")
|
||||
seq_lens = [30, 12]
|
||||
query_lens = [30, 12]
|
||||
spans = [[(4, 12), (20, 24)], []]
|
||||
query_start_loc, seq_lens_t, token_to_req, _, _ = make_batch(
|
||||
seq_lens, query_lens, device
|
||||
)
|
||||
num_tokens = int(query_start_loc[-1])
|
||||
|
||||
# CSR span layout: request -> contiguous [start, end) span list.
|
||||
indptr = [0, 2, 2]
|
||||
starts = [4, 20]
|
||||
ends = [12, 24]
|
||||
left = torch.zeros(num_tokens, dtype=torch.int32, device=device)
|
||||
right = torch.zeros(num_tokens, dtype=torch.int32, device=device)
|
||||
_compute_image_visibility_kernel[(num_tokens,)](
|
||||
left,
|
||||
right,
|
||||
torch.tensor(indptr, dtype=torch.int32, device=device),
|
||||
torch.tensor(starts, dtype=torch.int32, device=device),
|
||||
torch.tensor(ends, dtype=torch.int32, device=device),
|
||||
query_start_loc,
|
||||
seq_lens_t,
|
||||
token_to_req,
|
||||
MAX_IMG,
|
||||
token_offset=0,
|
||||
)
|
||||
ref_left, ref_right = ref_left_right(seq_lens, query_lens, spans, MAX_IMG)
|
||||
assert left.cpu().tolist() == ref_left
|
||||
assert right.cpu().tolist() == ref_right
|
||||
|
||||
|
||||
def combine_case(
|
||||
compress_ratio: int,
|
||||
topk: int,
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
spans: list[list[tuple[int, int]]],
|
||||
with_image: bool,
|
||||
):
|
||||
"""Run combine_topk_swa_indices and return (indices, lens, expected)."""
|
||||
device = torch.device("cuda")
|
||||
num_reqs = len(seq_lens)
|
||||
query_start_loc = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
query_start_loc[1:] = torch.tensor(
|
||||
query_lens, dtype=torch.int32, device=device
|
||||
).cumsum(0)
|
||||
num_tokens = int(query_start_loc[-1])
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device)
|
||||
gather_lens = torch.tensor(
|
||||
[q + min(s - q, WINDOW - 1) for s, q in zip(seq_lens, query_lens)],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
N = (max(seq_lens) + compress_ratio - 1) // compress_ratio
|
||||
M = N + int(gather_lens.max()) + 8
|
||||
gen = torch.Generator(device="cpu").manual_seed(0)
|
||||
topk_indices = torch.randint(
|
||||
0, 4096, (num_tokens, max(topk, 1)), generator=gen, dtype=torch.int32
|
||||
).to(device)
|
||||
topk_indices = topk_indices[:, : max(topk, 1)]
|
||||
|
||||
if with_image:
|
||||
lefts, rights = ref_left_right(seq_lens, query_lens, spans, MAX_IMG)
|
||||
left_t = torch.tensor(lefts, dtype=torch.int32, device=device)
|
||||
right_t = torch.tensor(rights, dtype=torch.int32, device=device)
|
||||
else:
|
||||
left_t = right_t = None
|
||||
|
||||
combined_indices, combined_lens = combine_topk_swa_indices(
|
||||
topk_indices,
|
||||
query_start_loc,
|
||||
seq_lens_t,
|
||||
gather_lens,
|
||||
WINDOW,
|
||||
compress_ratio,
|
||||
topk,
|
||||
M,
|
||||
N,
|
||||
left_visible=left_t,
|
||||
right_visible=right_t,
|
||||
max_image_tokens=MAX_IMG,
|
||||
)
|
||||
|
||||
# Reference rows.
|
||||
lefts, rights = ref_left_right(
|
||||
seq_lens,
|
||||
query_lens,
|
||||
spans if with_image else [[] for _ in seq_lens],
|
||||
MAX_IMG,
|
||||
)
|
||||
topk_cpu = topk_indices.cpu()
|
||||
width = WINDOW + MAX_IMG
|
||||
combined_topk = (topk + width + 127) // 128 * 128
|
||||
rows = []
|
||||
lens = []
|
||||
token = 0
|
||||
for b, (seq_len, query_len) in enumerate(zip(seq_lens, query_lens)):
|
||||
prefix_len = seq_len - query_len
|
||||
gather_start = seq_len - int(gather_lens[b])
|
||||
for i in range(query_len):
|
||||
pos = prefix_len + i
|
||||
topk_len = min((pos + 1) // compress_ratio, topk)
|
||||
start, end = ref_swa_bounds(pos, WINDOW, lefts[token], rights[token])
|
||||
swa_len = end - start
|
||||
row = [-1] * combined_topk
|
||||
for j in range(topk_len):
|
||||
row[j] = int(topk_cpu[token, j]) + M * b
|
||||
for j in range(swa_len):
|
||||
row[topk_len + j] = M * b + N + start + j - gather_start
|
||||
rows.append(row)
|
||||
lens.append(topk_len + swa_len)
|
||||
token += 1
|
||||
return combined_indices, combined_lens, rows, lens
|
||||
|
||||
|
||||
COMBINE_CASES = [
|
||||
dict(compress_ratio=1, topk=0), # SWA-only layer
|
||||
dict(compress_ratio=4, topk=16), # C4A layer
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("cfg", COMBINE_CASES)
|
||||
def test_combine_topk_swa_with_image_spans(cfg):
|
||||
case = CASES[0]
|
||||
indices, lens, rows, exp_lens = combine_case(
|
||||
cfg["compress_ratio"],
|
||||
cfg["topk"],
|
||||
case["seq_lens"],
|
||||
case["query_lens"],
|
||||
case["spans"],
|
||||
with_image=True,
|
||||
)
|
||||
assert lens.cpu().tolist() == exp_lens
|
||||
assert indices.cpu().tolist() == rows
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("cfg", COMBINE_CASES)
|
||||
def test_combine_topk_swa_without_image_unchanged(cfg):
|
||||
"""left_visible=None must reproduce the plain causal combined indices."""
|
||||
case = CASES[0]
|
||||
indices, lens, rows, exp_lens = combine_case(
|
||||
cfg["compress_ratio"],
|
||||
cfg["topk"],
|
||||
case["seq_lens"],
|
||||
case["query_lens"],
|
||||
case["spans"],
|
||||
with_image=False,
|
||||
)
|
||||
assert lens.cpu().tolist() == exp_lens
|
||||
assert indices.cpu().tolist() == rows
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_flashinfer_mixed_sparse_indices_with_image_spans():
|
||||
device = torch.device("cuda")
|
||||
# 1 decode token (req 0) + two prefill requests (reqs 1, 2).
|
||||
seq_lens = [20, 30, 9]
|
||||
query_lens = [1, 30, 9]
|
||||
spans = [[], [(4, 12)], [(0, 6)]]
|
||||
query_start_loc = torch.tensor([0, 1, 31, 40], dtype=torch.int32, device=device)
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device)
|
||||
token_to_req = torch.tensor(
|
||||
[0] + [1] * 30 + [2] * 9, dtype=torch.int32, device=device
|
||||
)
|
||||
block_table = torch.arange(3, dtype=torch.int32, device=device).view(3, 1)
|
||||
decode_swa = torch.tensor(
|
||||
[[100, 101, 102, 103, 104, 105, 106, 107]], dtype=torch.int32, device=device
|
||||
)
|
||||
prefill_topk = torch.zeros(39, 0, dtype=torch.int32, device=device)
|
||||
|
||||
lefts, rights = ref_left_right(seq_lens, query_lens, spans, MAX_IMG)
|
||||
left_t = torch.tensor(lefts, dtype=torch.int32, device=device)
|
||||
right_t = torch.tensor(rights, dtype=torch.int32, device=device)
|
||||
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=decode_swa,
|
||||
decode_compressed_indices=None,
|
||||
decode_compressed_topk_lens=None,
|
||||
prefill_topk_indices=prefill_topk,
|
||||
query_start_loc=query_start_loc,
|
||||
seq_lens=seq_lens_t,
|
||||
token_to_req_indices=token_to_req,
|
||||
swa_block_table=block_table,
|
||||
swa_block_size=BLOCK_SIZE,
|
||||
compressed_block_table=None,
|
||||
compressed_block_size=BLOCK_SIZE,
|
||||
window_size=WINDOW,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
prefill_left_visible=left_t,
|
||||
prefill_right_visible=right_t,
|
||||
max_image_tokens=MAX_IMG,
|
||||
)
|
||||
|
||||
swa_total = WINDOW + MAX_IMG
|
||||
assert sparse_indices.shape == (40, swa_total)
|
||||
assert sparse_lens.cpu().tolist() == [swa_total] * 40
|
||||
# Decode row: slots copied, image-extension columns padded with -1.
|
||||
decode_row = sparse_indices[0].cpu().tolist()
|
||||
assert decode_row[:WINDOW] == list(range(100, 108))
|
||||
assert decode_row[WINDOW:] == [-1] * MAX_IMG
|
||||
# Prefill rows: paged slot ids over the widened window.
|
||||
exp_rows, _ = ref_swa_slot_rows(
|
||||
seq_lens, query_lens, spans, block_table, WINDOW, MAX_IMG, swa_total
|
||||
)
|
||||
actual = sparse_indices[1:].cpu().tolist()
|
||||
assert actual == exp_rows[1:]
|
||||
|
||||
|
||||
def make_builder(vision: bool) -> DeepseekSparseSWAMetadataBuilder:
|
||||
overrides: dict = {"sliding_window": WINDOW}
|
||||
if vision:
|
||||
overrides.update(vision_n_layers=2, vision_max_n_token=MAX_IMG)
|
||||
vllm_config = create_vllm_config(
|
||||
max_model_len=4096,
|
||||
max_num_batched_tokens=64,
|
||||
max_num_seqs=8,
|
||||
hf_config_override=overrides,
|
||||
)
|
||||
spec = SlidingWindowMLASpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=WINDOW,
|
||||
cache_dtype_str="auto",
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
return DeepseekSparseSWAMetadataBuilder(
|
||||
kv_cache_spec=spec,
|
||||
layer_names=["layer0"],
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cuda"),
|
||||
)
|
||||
|
||||
|
||||
def build_metadata(
|
||||
builder: DeepseekSparseSWAMetadataBuilder,
|
||||
seq_lens: list[int],
|
||||
query_lens: list[int],
|
||||
mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None,
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch(
|
||||
seq_lens, query_lens, device
|
||||
)
|
||||
return builder.build(
|
||||
0,
|
||||
CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=seq_lens_t,
|
||||
seq_lens_cpu_upper_bound=seq_lens_t.cpu(),
|
||||
num_reqs=len(seq_lens),
|
||||
num_actual_tokens=int(query_start_loc[-1]),
|
||||
max_query_len=max(query_lens),
|
||||
max_seq_len=max(seq_lens),
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=slot_mapping,
|
||||
causal=True,
|
||||
mm_req_doc_ranges=mm_req_doc_ranges,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_builder_in_image_visibility():
|
||||
seq_lens = [30, 12]
|
||||
query_lens = [30, 12]
|
||||
spans = [[(4, 12), (20, 24)], []]
|
||||
builder = make_builder(vision=True)
|
||||
md = build_metadata(builder, seq_lens, query_lens, {0: spans[0], 1: spans[1]})
|
||||
assert md.num_prefills == 2
|
||||
assert md.num_decode_tokens == 0
|
||||
assert md.prefill_swa_indices.shape[-1] == WIDTH
|
||||
assert md.prefill_left_visible is not None
|
||||
assert md.prefill_right_visible is not None
|
||||
|
||||
ref_left, ref_right = ref_left_right(seq_lens, query_lens, spans, MAX_IMG)
|
||||
assert md.prefill_left_visible.cpu().tolist() == ref_left
|
||||
assert md.prefill_right_visible.cpu().tolist() == ref_right
|
||||
|
||||
_, _, _, _, block_table = make_batch(seq_lens, query_lens, torch.device("cuda"))
|
||||
rows, lens = ref_swa_slot_rows(
|
||||
seq_lens, query_lens, spans, block_table, WINDOW, MAX_IMG, WIDTH
|
||||
)
|
||||
assert md.prefill_swa_lens.cpu().tolist() == lens
|
||||
assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_builder_no_image_spans_fast_path():
|
||||
"""Vision model, image-free batch: no visibility tensors, plain window."""
|
||||
seq_lens = [30, 12]
|
||||
query_lens = [30, 12]
|
||||
builder = make_builder(vision=True)
|
||||
md = build_metadata(builder, seq_lens, query_lens, {0: [], 1: []})
|
||||
assert md.prefill_left_visible is None
|
||||
assert md.prefill_right_visible is None
|
||||
|
||||
_, _, _, _, block_table = make_batch(seq_lens, query_lens, torch.device("cuda"))
|
||||
rows, lens = ref_swa_slot_rows(
|
||||
seq_lens, query_lens, [[], []], block_table, WINDOW, MAX_IMG, WIDTH
|
||||
)
|
||||
assert md.prefill_swa_lens.cpu().tolist() == lens
|
||||
assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_builder_text_model_unchanged():
|
||||
"""Text-only model: buffers stay window-sized and spans are ignored."""
|
||||
seq_lens = [30, 12]
|
||||
query_lens = [30, 12]
|
||||
builder = make_builder(vision=False)
|
||||
assert builder.max_image_tokens == 0
|
||||
md = build_metadata(builder, seq_lens, query_lens, None)
|
||||
assert md.prefill_swa_indices.shape[-1] == WINDOW
|
||||
assert md.prefill_left_visible is None
|
||||
|
||||
_, _, _, _, block_table = make_batch(seq_lens, query_lens, torch.device("cuda"))
|
||||
rows, lens = ref_swa_slot_rows(
|
||||
seq_lens, query_lens, [[], []], block_table, WINDOW, MAX_IMG, WINDOW
|
||||
)
|
||||
assert md.prefill_swa_lens.cpu().tolist() == lens
|
||||
assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows
|
||||
Reference in New Issue
Block a user