[Bugfix][Model] Restore causal image SWA for DeepSeek V4.1 (#57152)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Codex
Isotr0py
parent
651a88c09a
commit
9f9e1dac26
@@ -462,13 +462,31 @@ def test_flashinfer_mixed_sparse_indices_with_image_spans():
|
|||||||
assert actual == exp_rows[1:]
|
assert actual == exp_rows[1:]
|
||||||
|
|
||||||
|
|
||||||
def make_builder(vision: bool) -> DeepseekSparseSWAMetadataBuilder:
|
def make_builder(
|
||||||
overrides: dict = {"sliding_window": WINDOW}
|
vision: bool,
|
||||||
|
window: int = WINDOW,
|
||||||
|
max_image_tokens: int = MAX_IMG,
|
||||||
|
max_num_batched_tokens: int = 64,
|
||||||
|
model_name: str = "meta-llama/Meta-Llama-3-8B",
|
||||||
|
builder_cls: type[
|
||||||
|
DeepseekSparseSWAMetadataBuilder
|
||||||
|
] = DeepseekSparseSWAMetadataBuilder,
|
||||||
|
mm_prefix_clamp_sliding_window: bool = True,
|
||||||
|
) -> DeepseekSparseSWAMetadataBuilder:
|
||||||
|
overrides: dict = {"sliding_window": window}
|
||||||
if vision:
|
if vision:
|
||||||
overrides.update(vision_n_layers=2, vision_max_n_token=MAX_IMG)
|
# Emulate the V4 vision config, whose mm_prefix_clamp_sliding_window
|
||||||
|
# gates the in-kernel SWA widening. V4.1 (causal image tokens) leaves
|
||||||
|
# it off.
|
||||||
|
overrides.update(
|
||||||
|
vision_n_layers=2,
|
||||||
|
vision_max_n_token=max_image_tokens,
|
||||||
|
mm_prefix_clamp_sliding_window=mm_prefix_clamp_sliding_window,
|
||||||
|
)
|
||||||
vllm_config = create_vllm_config(
|
vllm_config = create_vllm_config(
|
||||||
|
model_name=model_name,
|
||||||
max_model_len=4096,
|
max_model_len=4096,
|
||||||
max_num_batched_tokens=64,
|
max_num_batched_tokens=max_num_batched_tokens,
|
||||||
max_num_seqs=8,
|
max_num_seqs=8,
|
||||||
hf_config_override=overrides,
|
hf_config_override=overrides,
|
||||||
)
|
)
|
||||||
@@ -477,11 +495,11 @@ def make_builder(vision: bool) -> DeepseekSparseSWAMetadataBuilder:
|
|||||||
num_kv_heads=1,
|
num_kv_heads=1,
|
||||||
head_size=512,
|
head_size=512,
|
||||||
dtype=torch.bfloat16,
|
dtype=torch.bfloat16,
|
||||||
sliding_window=WINDOW,
|
sliding_window=window,
|
||||||
cache_dtype_str="auto",
|
cache_dtype_str="auto",
|
||||||
model_version="deepseek_v4",
|
model_version="deepseek_v4",
|
||||||
)
|
)
|
||||||
return DeepseekSparseSWAMetadataBuilder(
|
return builder_cls(
|
||||||
kv_cache_spec=spec,
|
kv_cache_spec=spec,
|
||||||
layer_names=["layer0"],
|
layer_names=["layer0"],
|
||||||
vllm_config=vllm_config,
|
vllm_config=vllm_config,
|
||||||
@@ -578,3 +596,60 @@ def test_builder_text_model_unchanged():
|
|||||||
)
|
)
|
||||||
assert md.prefill_swa_lens.cpu().tolist() == lens
|
assert md.prefill_swa_lens.cpu().tolist() == lens
|
||||||
assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows
|
assert md.prefill_swa_indices[:, 0].cpu().tolist() == rows
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||||
|
@pytest.mark.parametrize("compress_ratio", [0, 1, 2])
|
||||||
|
@pytest.mark.parametrize("query_len", [100, 4000])
|
||||||
|
def test_v41_image_prefill_uses_causal_swa(compress_ratio, query_len):
|
||||||
|
"""V4.1 image tokens use the same causal SWA as text, including in chunks."""
|
||||||
|
from vllm.models.deepseek_v41.common.ops.cache_utils import (
|
||||||
|
combine_topk_swa_indices as combine_v41,
|
||||||
|
)
|
||||||
|
from vllm.models.deepseek_v41.sparse_mla import DeepseekV41SparseSWAMetadataBuilder
|
||||||
|
|
||||||
|
seq_len, window, max_image_tokens = 4000, 128, 2048
|
||||||
|
spans = [(1900, 3947)]
|
||||||
|
builder = make_builder(
|
||||||
|
True,
|
||||||
|
window,
|
||||||
|
max_image_tokens,
|
||||||
|
query_len,
|
||||||
|
builder_cls=DeepseekV41SparseSWAMetadataBuilder,
|
||||||
|
mm_prefix_clamp_sliding_window=False,
|
||||||
|
)
|
||||||
|
assert builder.max_image_tokens == 0
|
||||||
|
md = build_metadata(builder, [seq_len], [query_len], {0: spans})
|
||||||
|
assert md.prefill_left_visible is None
|
||||||
|
assert md.prefill_right_visible is None
|
||||||
|
assert md.prefill_swa_indices.shape[-1] == window
|
||||||
|
plan = md.get_prefill_chunk_plan(
|
||||||
|
compress_ratio, prefill_chunk_size=4, has_compressed=compress_ratio > 0
|
||||||
|
)
|
||||||
|
[(start, end, n, m)] = plan
|
||||||
|
gather_len = int(md.prefill_gather_lens[0])
|
||||||
|
assert (start, end, m - n) == (0, 1, gather_len)
|
||||||
|
assert gather_len == query_len + min(seq_len - query_len, window - 1)
|
||||||
|
indices, lens = combine_v41(
|
||||||
|
torch.empty(query_len, 0, dtype=torch.int32, device="cuda"),
|
||||||
|
md.query_start_loc,
|
||||||
|
md.prefill_seq_lens,
|
||||||
|
md.prefill_gather_lens,
|
||||||
|
window,
|
||||||
|
compress_ratio,
|
||||||
|
0,
|
||||||
|
m,
|
||||||
|
n,
|
||||||
|
)
|
||||||
|
indices, lens = indices.cpu(), lens.cpu().tolist()
|
||||||
|
paged_indices = md.prefill_swa_indices.cpu()
|
||||||
|
paged_lens = md.prefill_swa_lens.cpu().tolist()
|
||||||
|
for token, pos in enumerate(range(seq_len - query_len, seq_len)):
|
||||||
|
lo, hi = max(0, pos - window + 1), pos + 1
|
||||||
|
row = indices[token, : lens[token]]
|
||||||
|
assert lens[token] == hi - lo
|
||||||
|
assert torch.all((row >= n) & (row < m))
|
||||||
|
absolute_keys = row - n + seq_len - gather_len
|
||||||
|
assert absolute_keys.tolist() == list(range(lo, hi))
|
||||||
|
assert paged_lens[token] == hi - lo
|
||||||
|
assert paged_indices[token, 0, : hi - lo].tolist() == list(range(lo, hi))
|
||||||
|
|||||||
@@ -288,13 +288,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
|||||||
self.n_groups = config.o_groups
|
self.n_groups = config.o_groups
|
||||||
self.n_local_groups = self.n_groups // tp_size
|
self.n_local_groups = self.n_groups // tp_size
|
||||||
self.window_size = config.sliding_window
|
self.window_size = config.sliding_window
|
||||||
# Vision variant: image spans are visible bidirectionally, widening
|
|
||||||
# prefill SWA index rows by up to max_image_tokens columns.
|
|
||||||
self.max_image_tokens = (
|
|
||||||
getattr(config, "vision_max_n_token", 0)
|
|
||||||
if getattr(config, "vision_n_layers", 0) > 0
|
|
||||||
else 0
|
|
||||||
)
|
|
||||||
# ---- v4.1 sparse-attention topology ----
|
# ---- v4.1 sparse-attention topology ----
|
||||||
# compress_ratios has one entry per layer (MTP layers included):
|
# compress_ratios has one entry per layer (MTP layers included):
|
||||||
# 0 = pure sliding window, 1 = full-length compressed cache,
|
# 0 = pure sliding window, 1 = full-length compressed cache,
|
||||||
@@ -576,7 +569,6 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
|||||||
_COMPUTE_SWA_INDICES_AND_LENS_KERNEL.register_warmup(
|
_COMPUTE_SWA_INDICES_AND_LENS_KERNEL.register_warmup(
|
||||||
window_size=self.window_size,
|
window_size=self.window_size,
|
||||||
block_size=self.swa_cache_layer.block_size,
|
block_size=self.swa_cache_layer.block_size,
|
||||||
max_image_tokens=self.max_image_tokens,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.compress_ratio > 1:
|
if self.compress_ratio > 1:
|
||||||
|
|||||||
@@ -815,16 +815,12 @@ def combine_topk_swa_indices(
|
|||||||
M: int,
|
M: int,
|
||||||
N: int,
|
N: int,
|
||||||
out: tuple[torch.Tensor, torch.Tensor] | None = None,
|
out: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||||
left_visible: torch.Tensor | None = None,
|
|
||||||
right_visible: torch.Tensor | None = None,
|
|
||||||
max_image_tokens: int = 0,
|
|
||||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
num_tokens = topk_indices.shape[0]
|
num_tokens = topk_indices.shape[0]
|
||||||
# max_image_tokens widens the SWA column region for in-image
|
# The width is fixed per model so the caller-provided workspace stays
|
||||||
# bidirectional visibility; the width is fixed per model so the
|
# valid on every batch.
|
||||||
# caller-provided workspace stays valid on image-free batches.
|
|
||||||
combined_topk = (
|
combined_topk = (
|
||||||
(topk + window_size + max_image_tokens + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1)
|
(topk + window_size + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1)
|
||||||
// _SPARSE_PREFILL_TOPK_ALIGNMENT
|
// _SPARSE_PREFILL_TOPK_ALIGNMENT
|
||||||
* _SPARSE_PREFILL_TOPK_ALIGNMENT
|
* _SPARSE_PREFILL_TOPK_ALIGNMENT
|
||||||
)
|
)
|
||||||
@@ -853,9 +849,6 @@ def combine_topk_swa_indices(
|
|||||||
TOP_K=topk,
|
TOP_K=topk,
|
||||||
COMPRESS_RATIO=compress_ratio,
|
COMPRESS_RATIO=compress_ratio,
|
||||||
WINDOW_SIZE=window_size,
|
WINDOW_SIZE=window_size,
|
||||||
left_visible=left_visible,
|
|
||||||
right_visible=right_visible,
|
|
||||||
max_image_tokens=max_image_tokens,
|
|
||||||
)
|
)
|
||||||
return combined_indices, combined_lens
|
return combined_indices, combined_lens
|
||||||
|
|
||||||
@@ -924,7 +917,6 @@ class CombineTopkSwaIndicesKernel(
|
|||||||
TOP_K: int
|
TOP_K: int
|
||||||
COMPRESS_RATIO: int
|
COMPRESS_RATIO: int
|
||||||
WINDOW_SIZE: int
|
WINDOW_SIZE: int
|
||||||
IMAGE_WIDTH: int
|
|
||||||
PADDED_TOP_K: int
|
PADDED_TOP_K: int
|
||||||
input_variant: TritonPointerInputVariant
|
input_variant: TritonPointerInputVariant
|
||||||
|
|
||||||
@@ -946,14 +938,11 @@ class CombineTopkSwaIndicesKernel(
|
|||||||
query_start_loc_ptr,
|
query_start_loc_ptr,
|
||||||
seq_lens_ptr,
|
seq_lens_ptr,
|
||||||
gather_lens_ptr,
|
gather_lens_ptr,
|
||||||
left_visible_ptr,
|
|
||||||
right_visible_ptr,
|
|
||||||
M,
|
M,
|
||||||
N,
|
N,
|
||||||
TOP_K: tl.constexpr,
|
TOP_K: tl.constexpr,
|
||||||
COMPRESS_RATIO: tl.constexpr,
|
COMPRESS_RATIO: tl.constexpr,
|
||||||
WINDOW_SIZE: tl.constexpr,
|
WINDOW_SIZE: tl.constexpr,
|
||||||
IMAGE_WIDTH: tl.constexpr,
|
|
||||||
PADDED_TOP_K: tl.constexpr,
|
PADDED_TOP_K: tl.constexpr,
|
||||||
):
|
):
|
||||||
batch_idx = tl.program_id(0)
|
batch_idx = tl.program_id(0)
|
||||||
@@ -988,19 +977,8 @@ class CombineTopkSwaIndicesKernel(
|
|||||||
# (integer div by constexpr 0 is UB and yields garbage, which
|
# (integer div by constexpr 0 is UB and yields garbage, which
|
||||||
# turns the store index below negative).
|
# turns the store index below negative).
|
||||||
topk_len = 0
|
topk_len = 0
|
||||||
if IMAGE_WIDTH > 0:
|
swa_start = tl.maximum(pos - (WINDOW_SIZE - 1), 0)
|
||||||
# In-image bidirectional visibility: the window starts up to
|
swa_len = pos - swa_start + 1
|
||||||
# max(left - (window - 1), 0) positions earlier and extends
|
|
||||||
# `right` positions past the query token (both 0 outside
|
|
||||||
# image spans, reducing to the plain causal window).
|
|
||||||
left = tl.load(left_visible_ptr + token_idx)
|
|
||||||
right = tl.load(right_visible_ptr + token_idx)
|
|
||||||
else:
|
|
||||||
left = 0
|
|
||||||
right = 0
|
|
||||||
left_add = tl.maximum(left - (WINDOW_SIZE - 1), 0)
|
|
||||||
swa_start = tl.maximum(pos - (WINDOW_SIZE - 1) - left_add, 0)
|
|
||||||
swa_len = pos + right - swa_start + 1
|
|
||||||
|
|
||||||
offset = tl.arange(0, PADDED_TOP_K)
|
offset = tl.arange(0, PADDED_TOP_K)
|
||||||
mask = offset < topk_len
|
mask = offset < topk_len
|
||||||
@@ -1014,18 +992,17 @@ class CombineTopkSwaIndicesKernel(
|
|||||||
mask=mask,
|
mask=mask,
|
||||||
)
|
)
|
||||||
# Index into gathered buffer: N + (position - gather_start)
|
# Index into gathered buffer: N + (position - gather_start)
|
||||||
# For positions [swa_start, pos + right], the buffer indices are:
|
# For positions [swa_start, pos], the buffer indices are:
|
||||||
# [N + swa_start - gather_start, N + pos + right - gather_start]
|
# [N + swa_start - gather_start, N + pos - gather_start]
|
||||||
for i in range(0, WINDOW_SIZE + IMAGE_WIDTH, WINDOW_SIZE):
|
swa_offset = tl.arange(0, WINDOW_SIZE)
|
||||||
swa_offset = i + tl.arange(0, WINDOW_SIZE)
|
tl.store(
|
||||||
tl.store(
|
combined_indices_ptr
|
||||||
combined_indices_ptr
|
+ token_idx * combined_indices_stride
|
||||||
+ token_idx * combined_indices_stride
|
+ topk_len
|
||||||
+ topk_len
|
+ swa_offset,
|
||||||
+ swa_offset,
|
M * batch_idx + N + swa_start + swa_offset - gather_start,
|
||||||
M * batch_idx + N + swa_start + swa_offset - gather_start,
|
mask=swa_offset < swa_len,
|
||||||
mask=swa_offset < swa_len,
|
)
|
||||||
)
|
|
||||||
|
|
||||||
combined_len = topk_len + swa_len
|
combined_len = topk_len + swa_len
|
||||||
tl.store(combined_lens_ptr + token_idx, combined_len)
|
tl.store(combined_lens_ptr + token_idx, combined_len)
|
||||||
@@ -1041,7 +1018,6 @@ class CombineTopkSwaIndicesKernel(
|
|||||||
topk: int,
|
topk: int,
|
||||||
compress_ratio: int,
|
compress_ratio: int,
|
||||||
WINDOW_SIZE: int,
|
WINDOW_SIZE: int,
|
||||||
image_width: int = 0,
|
|
||||||
) -> CompileKey:
|
) -> CompileKey:
|
||||||
padded_topk = next_power_of_2(topk_width)
|
padded_topk = next_power_of_2(topk_width)
|
||||||
input_variant = TritonPointerInputVariant.from_alignment(
|
input_variant = TritonPointerInputVariant.from_alignment(
|
||||||
@@ -1054,7 +1030,6 @@ class CombineTopkSwaIndicesKernel(
|
|||||||
TOP_K=topk,
|
TOP_K=topk,
|
||||||
COMPRESS_RATIO=compress_ratio,
|
COMPRESS_RATIO=compress_ratio,
|
||||||
WINDOW_SIZE=WINDOW_SIZE,
|
WINDOW_SIZE=WINDOW_SIZE,
|
||||||
IMAGE_WIDTH=image_width,
|
|
||||||
PADDED_TOP_K=padded_topk,
|
PADDED_TOP_K=padded_topk,
|
||||||
input_variant=input_variant,
|
input_variant=input_variant,
|
||||||
)
|
)
|
||||||
@@ -1064,19 +1039,10 @@ class CombineTopkSwaIndicesKernel(
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
window_size = _hf_config_int(vllm_config, "sliding_window", 128)
|
window_size = _hf_config_int(vllm_config, "sliding_window", 128)
|
||||||
image_width = (
|
|
||||||
_hf_config_int(vllm_config, "vision_max_n_token", 0)
|
|
||||||
if _hf_config_int(vllm_config, "vision_n_layers", 0) > 0
|
|
||||||
else 0
|
|
||||||
)
|
|
||||||
# Warm both the plain-window variant (batches without image spans)
|
|
||||||
# and the in-image bidirectional variant.
|
|
||||||
image_widths = [0, image_width] if image_width > 0 else [0]
|
|
||||||
return self._trace_dispatch(self.dispatch)(
|
return self._trace_dispatch(self.dispatch)(
|
||||||
_combine_topk_swa_warmup_inputs(vllm_config),
|
_combine_topk_swa_warmup_inputs(vllm_config),
|
||||||
_COMBINE_TOPK_SWA_POINTER_INPUTS,
|
_COMBINE_TOPK_SWA_POINTER_INPUTS,
|
||||||
WINDOW_SIZE=window_size,
|
WINDOW_SIZE=window_size,
|
||||||
image_width=image_widths,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def compile(self, compile_key: CompileKey) -> None:
|
def compile(self, compile_key: CompileKey) -> None:
|
||||||
@@ -1093,14 +1059,11 @@ class CombineTopkSwaIndicesKernel(
|
|||||||
input_variant.pointer("query_start_loc", torch.int32),
|
input_variant.pointer("query_start_loc", torch.int32),
|
||||||
input_variant.pointer("seq_lens", torch.int32),
|
input_variant.pointer("seq_lens", torch.int32),
|
||||||
input_variant.pointer("gather_lens", torch.int32),
|
input_variant.pointer("gather_lens", torch.int32),
|
||||||
int32_ptr,
|
|
||||||
int32_ptr,
|
|
||||||
1, # do not specialize M
|
1, # do not specialize M
|
||||||
1, # do not specialize N
|
1, # do not specialize N
|
||||||
TOP_K=compile_key.TOP_K,
|
TOP_K=compile_key.TOP_K,
|
||||||
COMPRESS_RATIO=compile_key.COMPRESS_RATIO,
|
COMPRESS_RATIO=compile_key.COMPRESS_RATIO,
|
||||||
WINDOW_SIZE=compile_key.WINDOW_SIZE,
|
WINDOW_SIZE=compile_key.WINDOW_SIZE,
|
||||||
IMAGE_WIDTH=compile_key.IMAGE_WIDTH,
|
|
||||||
PADDED_TOP_K=compile_key.PADDED_TOP_K,
|
PADDED_TOP_K=compile_key.PADDED_TOP_K,
|
||||||
grid=(1, _COMBINE_TOPK_SWA_NUM_WORKERS),
|
grid=(1, _COMBINE_TOPK_SWA_NUM_WORKERS),
|
||||||
)
|
)
|
||||||
@@ -1119,13 +1082,8 @@ class CombineTopkSwaIndicesKernel(
|
|||||||
TOP_K: int,
|
TOP_K: int,
|
||||||
COMPRESS_RATIO: int,
|
COMPRESS_RATIO: int,
|
||||||
WINDOW_SIZE: int,
|
WINDOW_SIZE: int,
|
||||||
left_visible: torch.Tensor | None = None,
|
|
||||||
right_visible: torch.Tensor | None = None,
|
|
||||||
max_image_tokens: int = 0,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
num_reqs = seq_lens.shape[0]
|
num_reqs = seq_lens.shape[0]
|
||||||
has_image = left_visible is not None
|
|
||||||
image_width = max_image_tokens if has_image else 0
|
|
||||||
self.kernel[(num_reqs, _COMBINE_TOPK_SWA_NUM_WORKERS)](
|
self.kernel[(num_reqs, _COMBINE_TOPK_SWA_NUM_WORKERS)](
|
||||||
combined_indices,
|
combined_indices,
|
||||||
combined_indices.stride(0),
|
combined_indices.stride(0),
|
||||||
@@ -1135,14 +1093,11 @@ class CombineTopkSwaIndicesKernel(
|
|||||||
query_start_loc,
|
query_start_loc,
|
||||||
seq_lens,
|
seq_lens,
|
||||||
gather_lens,
|
gather_lens,
|
||||||
left_visible if has_image else topk_indices,
|
|
||||||
right_visible if has_image else topk_indices,
|
|
||||||
M,
|
M,
|
||||||
N,
|
N,
|
||||||
TOP_K=TOP_K,
|
TOP_K=TOP_K,
|
||||||
COMPRESS_RATIO=COMPRESS_RATIO,
|
COMPRESS_RATIO=COMPRESS_RATIO,
|
||||||
WINDOW_SIZE=WINDOW_SIZE,
|
WINDOW_SIZE=WINDOW_SIZE,
|
||||||
IMAGE_WIDTH=image_width,
|
|
||||||
PADDED_TOP_K=next_power_of_2(topk_indices.shape[-1]),
|
PADDED_TOP_K=next_power_of_2(topk_indices.shape[-1]),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1169,9 +1124,6 @@ def build_flashinfer_mixed_sparse_indices(
|
|||||||
decode_is_valid_token: torch.Tensor | None = None,
|
decode_is_valid_token: torch.Tensor | None = None,
|
||||||
swa_block_span: int | None = None,
|
swa_block_span: int | None = None,
|
||||||
compressed_block_span: int | None = None,
|
compressed_block_span: int | None = None,
|
||||||
prefill_left_visible: torch.Tensor | None = None,
|
|
||||||
prefill_right_visible: torch.Tensor | None = None,
|
|
||||||
max_image_tokens: int = 0,
|
|
||||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||||
"""Build the FlashInfer DSV4 sparse-index matrix for decode-first batches.
|
"""Build the FlashInfer DSV4 sparse-index matrix for decode-first batches.
|
||||||
|
|
||||||
@@ -1181,21 +1133,11 @@ def build_flashinfer_mixed_sparse_indices(
|
|||||||
per token). Decode tokens read precomputed SWA/compressed indices; prefill
|
per token). Decode tokens read precomputed SWA/compressed indices; prefill
|
||||||
tokens derive their SWA window from the position and translate local
|
tokens derive their SWA window from the position and translate local
|
||||||
compressed indices to global slots via the block tables.
|
compressed indices to global slots via the block tables.
|
||||||
|
|
||||||
When ``prefill_left_visible``/``prefill_right_visible`` are given (vision
|
|
||||||
variant), the SWA column region widens by ``max_image_tokens`` and prefill
|
|
||||||
tokens inside an image span get a bidirectionally widened window; decode
|
|
||||||
rows are padded with -1 across the extra columns.
|
|
||||||
"""
|
"""
|
||||||
assert decode_swa_indices.dtype == torch.int32
|
assert decode_swa_indices.dtype == torch.int32
|
||||||
assert decode_swa_indices.dim() == 2
|
assert decode_swa_indices.dim() == 2
|
||||||
swa_index_width = decode_swa_indices.shape[-1]
|
swa_total_width = decode_swa_indices.shape[-1]
|
||||||
assert swa_index_width >= window_size
|
assert swa_total_width >= window_size
|
||||||
has_image = prefill_left_visible is not None
|
|
||||||
image_width = max_image_tokens if has_image else 0
|
|
||||||
swa_total_width = swa_index_width + image_width
|
|
||||||
if has_image:
|
|
||||||
assert prefill_right_visible is not None
|
|
||||||
if decode_compressed_topk_lens is not None:
|
if decode_compressed_topk_lens is not None:
|
||||||
assert decode_compressed_topk_lens.dtype == torch.int32
|
assert decode_compressed_topk_lens.dtype == torch.int32
|
||||||
assert prefill_topk_indices.dtype == torch.int32
|
assert prefill_topk_indices.dtype == torch.int32
|
||||||
@@ -1254,7 +1196,7 @@ def build_flashinfer_mixed_sparse_indices(
|
|||||||
if num_tokens == 0:
|
if num_tokens == 0:
|
||||||
return sparse_indices, sparse_topk_lens
|
return sparse_indices, sparse_topk_lens
|
||||||
|
|
||||||
window_block_size = triton.next_power_of_2(max(swa_index_width, 1))
|
window_block_size = triton.next_power_of_2(max(swa_total_width, 1))
|
||||||
topk_block_size = triton.next_power_of_2(max(padded_topk, 1))
|
topk_block_size = triton.next_power_of_2(max(padded_topk, 1))
|
||||||
max_block_size = max(window_block_size, topk_block_size)
|
max_block_size = max(window_block_size, topk_block_size)
|
||||||
num_warps = 4 if max_block_size >= 256 else 1
|
num_warps = 4 if max_block_size >= 256 else 1
|
||||||
@@ -1278,8 +1220,6 @@ def build_flashinfer_mixed_sparse_indices(
|
|||||||
decode_is_valid_token,
|
decode_is_valid_token,
|
||||||
prefill_topk_indices,
|
prefill_topk_indices,
|
||||||
prefill_topk_indices.stride(0),
|
prefill_topk_indices.stride(0),
|
||||||
prefill_left_visible if has_image else token_to_req_indices,
|
|
||||||
prefill_right_visible if has_image else token_to_req_indices,
|
|
||||||
query_start_loc,
|
query_start_loc,
|
||||||
seq_lens,
|
seq_lens,
|
||||||
token_to_req_indices,
|
token_to_req_indices,
|
||||||
@@ -1293,8 +1233,7 @@ def build_flashinfer_mixed_sparse_indices(
|
|||||||
compressed_span,
|
compressed_span,
|
||||||
NUM_DECODE_TOKENS=num_decode_tokens,
|
NUM_DECODE_TOKENS=num_decode_tokens,
|
||||||
WINDOW_SIZE=window_size,
|
WINDOW_SIZE=window_size,
|
||||||
SWA_INDEX_WIDTH=swa_index_width,
|
SWA_INDEX_WIDTH=swa_total_width,
|
||||||
IMAGE_WIDTH=image_width,
|
|
||||||
COMPRESS_RATIO=compress_ratio,
|
COMPRESS_RATIO=compress_ratio,
|
||||||
TOP_K=topk,
|
TOP_K=topk,
|
||||||
PADDED_TOP_K=padded_topk,
|
PADDED_TOP_K=padded_topk,
|
||||||
@@ -1349,8 +1288,6 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
|||||||
decode_is_valid_token_ptr,
|
decode_is_valid_token_ptr,
|
||||||
prefill_topk_indices_ptr,
|
prefill_topk_indices_ptr,
|
||||||
prefill_topk_stride,
|
prefill_topk_stride,
|
||||||
left_visible_ptr,
|
|
||||||
right_visible_ptr,
|
|
||||||
query_start_loc_ptr,
|
query_start_loc_ptr,
|
||||||
seq_lens_ptr,
|
seq_lens_ptr,
|
||||||
token_to_req_indices_ptr,
|
token_to_req_indices_ptr,
|
||||||
@@ -1365,7 +1302,6 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
|||||||
NUM_DECODE_TOKENS,
|
NUM_DECODE_TOKENS,
|
||||||
WINDOW_SIZE: tl.constexpr,
|
WINDOW_SIZE: tl.constexpr,
|
||||||
SWA_INDEX_WIDTH: tl.constexpr,
|
SWA_INDEX_WIDTH: tl.constexpr,
|
||||||
IMAGE_WIDTH: tl.constexpr,
|
|
||||||
COMPRESS_RATIO: tl.constexpr,
|
COMPRESS_RATIO: tl.constexpr,
|
||||||
TOP_K: tl.constexpr,
|
TOP_K: tl.constexpr,
|
||||||
PADDED_TOP_K: tl.constexpr,
|
PADDED_TOP_K: tl.constexpr,
|
||||||
@@ -1377,18 +1313,14 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
|||||||
TOPK_BLOCK_SIZE: tl.constexpr,
|
TOPK_BLOCK_SIZE: tl.constexpr,
|
||||||
):
|
):
|
||||||
token_idx = tl.program_id(0)
|
token_idx = tl.program_id(0)
|
||||||
# Total SWA column count; > SWA_INDEX_WIDTH only for in-image
|
|
||||||
# bidirectional visibility (vision variant), where the extra columns are
|
|
||||||
# -1-padded for decode rows.
|
|
||||||
SWA_TOTAL_WIDTH: tl.constexpr = SWA_INDEX_WIDTH + IMAGE_WIDTH
|
|
||||||
|
|
||||||
if token_idx < NUM_DECODE_TOKENS:
|
if token_idx < NUM_DECODE_TOKENS:
|
||||||
for i in range(0, SWA_TOTAL_WIDTH, WINDOW_BLOCK_SIZE):
|
for i in range(0, SWA_INDEX_WIDTH, WINDOW_BLOCK_SIZE):
|
||||||
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
|
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
|
||||||
mask = offset < SWA_TOTAL_WIDTH
|
mask = offset < SWA_INDEX_WIDTH
|
||||||
values = tl.load(
|
values = tl.load(
|
||||||
decode_swa_indices_ptr + token_idx * decode_swa_stride + offset,
|
decode_swa_indices_ptr + token_idx * decode_swa_stride + offset,
|
||||||
mask=offset < SWA_INDEX_WIDTH,
|
mask=mask,
|
||||||
other=-1,
|
other=-1,
|
||||||
)
|
)
|
||||||
values = _remap_flashinfer_index(values, swa_block_size, swa_block_span)
|
values = _remap_flashinfer_index(values, swa_block_size, swa_block_span)
|
||||||
@@ -1431,7 +1363,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
|||||||
tl.store(
|
tl.store(
|
||||||
sparse_indices_ptr
|
sparse_indices_ptr
|
||||||
+ token_idx * sparse_indices_stride
|
+ token_idx * sparse_indices_stride
|
||||||
+ SWA_TOTAL_WIDTH
|
+ SWA_INDEX_WIDTH
|
||||||
+ offset,
|
+ offset,
|
||||||
values,
|
values,
|
||||||
mask=mask,
|
mask=mask,
|
||||||
@@ -1445,7 +1377,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
|||||||
else:
|
else:
|
||||||
compressed_len = tl.full((), DECODE_COMPRESSED_TOPK, dtype=tl.int32)
|
compressed_len = tl.full((), DECODE_COMPRESSED_TOPK, dtype=tl.int32)
|
||||||
|
|
||||||
tl.store(sparse_topk_lens_ptr + token_idx, SWA_TOTAL_WIDTH + compressed_len)
|
tl.store(sparse_topk_lens_ptr + token_idx, SWA_INDEX_WIDTH + compressed_len)
|
||||||
return
|
return
|
||||||
|
|
||||||
prefill_idx = token_idx - NUM_DECODE_TOKENS
|
prefill_idx = token_idx - NUM_DECODE_TOKENS
|
||||||
@@ -1457,26 +1389,17 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
|||||||
start_pos = seq_len - query_len
|
start_pos = seq_len - query_len
|
||||||
token_idx_in_query = token_idx - query_start
|
token_idx_in_query = token_idx - query_start
|
||||||
pos = start_pos + token_idx_in_query
|
pos = start_pos + token_idx_in_query
|
||||||
if IMAGE_WIDTH > 0:
|
swa_start_pos = tl.maximum(pos - (WINDOW_SIZE - 1), 0)
|
||||||
# In-image bidirectional visibility: window starts up to
|
swa_len = pos - swa_start_pos + 1
|
||||||
# max(left - (window - 1), 0) earlier and extends `right` past pos.
|
|
||||||
left = tl.load(left_visible_ptr + token_idx)
|
|
||||||
right = tl.load(right_visible_ptr + token_idx)
|
|
||||||
else:
|
|
||||||
left = 0
|
|
||||||
right = 0
|
|
||||||
left_add = tl.maximum(left - (WINDOW_SIZE - 1), 0)
|
|
||||||
swa_start_pos = tl.maximum(pos - (WINDOW_SIZE - 1) - left_add, 0)
|
|
||||||
swa_len = pos + right - swa_start_pos + 1
|
|
||||||
if COMPRESS_RATIO > 0:
|
if COMPRESS_RATIO > 0:
|
||||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
|
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
|
||||||
else:
|
else:
|
||||||
# SWA-only layers (ratio 0): no indexer topk; div by constexpr 0 is UB.
|
# SWA-only layers (ratio 0): no indexer topk; div by constexpr 0 is UB.
|
||||||
topk_len = 0
|
topk_len = 0
|
||||||
|
|
||||||
for i in range(0, SWA_TOTAL_WIDTH, WINDOW_BLOCK_SIZE):
|
for i in range(0, SWA_INDEX_WIDTH, WINDOW_BLOCK_SIZE):
|
||||||
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
|
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
|
||||||
mask = offset < SWA_TOTAL_WIDTH
|
mask = offset < SWA_INDEX_WIDTH
|
||||||
pos_offset = swa_start_pos + offset
|
pos_offset = swa_start_pos + offset
|
||||||
block_indices = pos_offset // swa_block_size
|
block_indices = pos_offset // swa_block_size
|
||||||
block_numbers = tl.load(
|
block_numbers = tl.load(
|
||||||
@@ -1520,10 +1443,10 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
|||||||
tl.store(
|
tl.store(
|
||||||
sparse_indices_ptr
|
sparse_indices_ptr
|
||||||
+ token_idx * sparse_indices_stride
|
+ token_idx * sparse_indices_stride
|
||||||
+ SWA_TOTAL_WIDTH
|
+ SWA_INDEX_WIDTH
|
||||||
+ offset,
|
+ offset,
|
||||||
slot_ids,
|
slot_ids,
|
||||||
mask=mask,
|
mask=mask,
|
||||||
)
|
)
|
||||||
|
|
||||||
tl.store(sparse_topk_lens_ptr + token_idx, SWA_TOTAL_WIDTH + topk_len)
|
tl.store(sparse_topk_lens_ptr + token_idx, SWA_INDEX_WIDTH + topk_len)
|
||||||
|
|||||||
@@ -311,7 +311,7 @@ class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention):
|
|||||||
else:
|
else:
|
||||||
assert self.topk_indices_buffer is not None
|
assert self.topk_indices_buffer is not None
|
||||||
top_k = self.topk_indices_buffer.shape[-1]
|
top_k = self.topk_indices_buffer.shape[-1]
|
||||||
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
|
combined_topk = round_up(top_k + self.window_size, 128)
|
||||||
current_workspace_manager().get_simultaneous(
|
current_workspace_manager().get_simultaneous(
|
||||||
((self.PREFILL_CHUNK_SIZE, m, q.shape[-1]), torch.bfloat16),
|
((self.PREFILL_CHUNK_SIZE, m, q.shape[-1]), torch.bfloat16),
|
||||||
((self.max_num_batched_tokens, combined_topk), torch.int32),
|
((self.max_num_batched_tokens, combined_topk), torch.int32),
|
||||||
@@ -411,7 +411,7 @@ class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention):
|
|||||||
)
|
)
|
||||||
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
|
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
|
||||||
workspace_manager = current_workspace_manager()
|
workspace_manager = current_workspace_manager()
|
||||||
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
|
combined_topk = round_up(top_k + self.window_size, 128)
|
||||||
for chunk_start, chunk_end, chunk_n, chunk_m in chunk_plan:
|
for chunk_start, chunk_end, chunk_n, chunk_m in chunk_plan:
|
||||||
chunk_size = chunk_end - chunk_start
|
chunk_size = chunk_end - chunk_start
|
||||||
kv_ws, idx_ws, lens_ws = workspace_manager.get_simultaneous(
|
kv_ws, idx_ws, lens_ws = workspace_manager.get_simultaneous(
|
||||||
@@ -457,21 +457,6 @@ class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention):
|
|||||||
chunk_m,
|
chunk_m,
|
||||||
chunk_n,
|
chunk_n,
|
||||||
out=(idx_ws[: qe - qs], lens_ws[: qe - qs]),
|
out=(idx_ws[: qe - qs], lens_ws[: qe - qs]),
|
||||||
left_visible=(
|
|
||||||
swa_metadata.prefill_left_visible[
|
|
||||||
num_decode_tokens + qs : num_decode_tokens + qe
|
|
||||||
]
|
|
||||||
if swa_metadata.prefill_left_visible is not None
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
right_visible=(
|
|
||||||
swa_metadata.prefill_right_visible[
|
|
||||||
num_decode_tokens + qs : num_decode_tokens + qe
|
|
||||||
]
|
|
||||||
if swa_metadata.prefill_right_visible is not None
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
max_image_tokens=self.max_image_tokens,
|
|
||||||
)
|
)
|
||||||
chunk_out = _token_slice(out, token_base + qs, token_base + qe)
|
chunk_out = _token_slice(out, token_base + qs, token_base + qe)
|
||||||
# Mega attention over the gathered non-paged bf16 KV (RoPE already
|
# Mega attention over the gathered non-paged bf16 KV (RoPE already
|
||||||
|
|||||||
@@ -417,10 +417,6 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
|||||||
decode_is_valid_token=decode_is_valid_token,
|
decode_is_valid_token=decode_is_valid_token,
|
||||||
swa_block_span=swa_block_span,
|
swa_block_span=swa_block_span,
|
||||||
compressed_block_span=compressed_block_span,
|
compressed_block_span=compressed_block_span,
|
||||||
prefill_left_visible=swa_metadata.prefill_left_visible,
|
|
||||||
prefill_right_visible=swa_metadata.prefill_right_visible,
|
|
||||||
# getattr for tests that bypass __init__ via object.__new__.
|
|
||||||
max_image_tokens=getattr(self, "max_image_tokens", 0),
|
|
||||||
)
|
)
|
||||||
if swa_only:
|
if swa_only:
|
||||||
swa_metadata.flashinfer_sparse_index_cache["swa_only"] = (
|
swa_metadata.flashinfer_sparse_index_cache["swa_only"] = (
|
||||||
|
|||||||
@@ -120,9 +120,7 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
|
|||||||
else:
|
else:
|
||||||
assert self.topk_indices_buffer is not None
|
assert self.topk_indices_buffer is not None
|
||||||
top_k = self.topk_indices_buffer.shape[-1]
|
top_k = self.topk_indices_buffer.shape[-1]
|
||||||
combined_topk = round_up(
|
combined_topk = round_up(top_k + self.window_size, 128)
|
||||||
top_k + self.window_size + self.max_image_tokens, 128
|
|
||||||
)
|
|
||||||
current_workspace_manager().get_simultaneous(
|
current_workspace_manager().get_simultaneous(
|
||||||
((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16),
|
((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16),
|
||||||
((self.max_num_batched_tokens, combined_topk), torch.int32),
|
((self.max_num_batched_tokens, combined_topk), torch.int32),
|
||||||
@@ -303,7 +301,7 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
|
|||||||
)
|
)
|
||||||
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
|
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
|
||||||
workspace_manager = current_workspace_manager()
|
workspace_manager = current_workspace_manager()
|
||||||
combined_topk = round_up(top_k + self.window_size + self.max_image_tokens, 128)
|
combined_topk = round_up(top_k + self.window_size, 128)
|
||||||
for chunk_start, chunk_end, chunk_N, chunk_M in chunk_plan:
|
for chunk_start, chunk_end, chunk_N, chunk_M in chunk_plan:
|
||||||
chunk_size = chunk_end - chunk_start
|
chunk_size = chunk_end - chunk_start
|
||||||
workspace = workspace_manager.get_simultaneous(
|
workspace = workspace_manager.get_simultaneous(
|
||||||
@@ -361,21 +359,6 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
|
|||||||
chunk_M,
|
chunk_M,
|
||||||
chunk_N,
|
chunk_N,
|
||||||
out=(combined_indices_out, combined_lens_out),
|
out=(combined_indices_out, combined_lens_out),
|
||||||
left_visible=(
|
|
||||||
swa_metadata.prefill_left_visible[
|
|
||||||
num_decode_tokens + query_start : num_decode_tokens + query_end
|
|
||||||
]
|
|
||||||
if swa_metadata.prefill_left_visible is not None
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
right_visible=(
|
|
||||||
swa_metadata.prefill_right_visible[
|
|
||||||
num_decode_tokens + query_start : num_decode_tokens + query_end
|
|
||||||
]
|
|
||||||
if swa_metadata.prefill_right_visible is not None
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
max_image_tokens=self.max_image_tokens,
|
|
||||||
)
|
)
|
||||||
flash_mla_sparse_fwd(
|
flash_mla_sparse_fwd(
|
||||||
q=q[query_start:query_end],
|
q=q[query_start:query_end],
|
||||||
|
|||||||
@@ -60,11 +60,3 @@ class DeepseekV41Config(PretrainedConfig):
|
|||||||
self.vision_max_n_token = vision_config.get("max_image_tokens", 1024)
|
self.vision_max_n_token = vision_config.get("max_image_tokens", 1024)
|
||||||
self.vision_min_pixels = vision_config.get("min_pixels", 295936)
|
self.vision_min_pixels = vision_config.get("min_pixels", 295936)
|
||||||
self.vision_max_wh_ratio = vision_config.get("max_wh_ratio")
|
self.vision_max_wh_ratio = vision_config.get("max_wh_ratio")
|
||||||
# The vision variant needs the mm-prefix plumbing (atomic image-span
|
|
||||||
# prefill + sparse-SWA window widening); the base arch-config
|
|
||||||
# convertor reads this config-level attribute.
|
|
||||||
self.is_mm_prefix_lm = vision_n_layers > 0
|
|
||||||
# The sparse-SWA index kernels widen the window within image spans
|
|
||||||
# in-kernel, so mm-prefix ranges longer than sliding_window must be
|
|
||||||
# kept (they are only consumed by those kernels).
|
|
||||||
self.mm_prefix_clamp_sliding_window = vision_n_layers > 0
|
|
||||||
|
|||||||
@@ -452,13 +452,15 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
|||||||
assert hasattr(hf_config, "sliding_window")
|
assert hasattr(hf_config, "sliding_window")
|
||||||
self.window_size = hf_config.sliding_window
|
self.window_size = hf_config.sliding_window
|
||||||
|
|
||||||
# Vision variant: image spans (up to vision_max_n_token tokens) are
|
# V4 vision variant: image spans (up to vision_max_n_token tokens) are
|
||||||
# visible bidirectionally, so prefill index rows widen from
|
# visible bidirectionally, so prefill index rows widen from
|
||||||
# window_size to window_size + max_image_tokens. Text-only models keep
|
# window_size to window_size + max_image_tokens. The V4 config sets
|
||||||
# max_image_tokens == 0 and take the original code paths everywhere.
|
# mm_prefix_clamp_sliding_window exactly for these in-kernel-widened
|
||||||
|
# ranges; V4.1 image tokens use the plain causal window, and text-only
|
||||||
|
# models keep max_image_tokens == 0 everywhere.
|
||||||
self.max_image_tokens = (
|
self.max_image_tokens = (
|
||||||
getattr(hf_config, "vision_max_n_token", 0)
|
getattr(hf_config, "vision_max_n_token", 0)
|
||||||
if getattr(hf_config, "vision_n_layers", 0) > 0
|
if getattr(hf_config, "mm_prefix_clamp_sliding_window", False)
|
||||||
else 0
|
else 0
|
||||||
)
|
)
|
||||||
self.prefill_index_width = self.window_size + self.max_image_tokens
|
self.prefill_index_width = self.window_size + self.max_image_tokens
|
||||||
|
|||||||
Reference in New Issue
Block a user