[Qwen3.8-Flash-Next] Fuse main QK-norm/RoPE/gate and KV-cache write into the QSA pre-indexer launch (#57097)

Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com>
Signed-off-by: Thien <thien@inferact.ai>
Co-authored-by: Thien <thien@inferact.ai>
This commit is contained in:
Shuolei Wang
2026-10-01 14:21:06 +08:00
committed by GitHub
co-authored by Thien
parent 70be6aae19
commit c9578f10ab
6 changed files with 325 additions and 47 deletions
+1 -1
View File
@@ -178,7 +178,7 @@ steps:
- vllm/v1/attention/backends/short_conv_attn.py
- tests/models/qwen4_exp/
commands:
- pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_pre_indexer.py models/qwen4_exp/test_qsa_reference.py
- pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_prepare.py models/qwen4_exp/test_qsa_reference.py
mirror:
amd:
label: ":amd: (MI355 DPX) Qwen4 Exp"
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the fused QSA pre-indexer."""
"""Correctness tests for the fused QSA prepare kernel."""
import pytest
import torch
@@ -15,9 +15,7 @@ from vllm.models.qwen4_exp.nvidia.ops.qsa import (
qsa_compress_groups_with_ratio,
qsa_store_cache_rows,
)
from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import (
qsa_pre_indexer,
)
from vllm.models.qwen4_exp.nvidia.ops.qsa_prepare import qsa_prepare
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
@@ -36,6 +34,7 @@ ROPE_POS_OFFSET = D
RTOL = 1.6e-2
ATOL = 1e-2
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16
def _make_block_table(block_counts):
@@ -61,6 +60,84 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
def _make_main_inputs(num_tokens: int, fp8_cache: bool) -> dict:
"""Random main-attention arguments with a paged cache in vLLM's layout."""
num_blocks = num_tokens // MAIN_PAGE + 2
cache = torch.zeros(
num_blocks,
MAIN_HK,
MAIN_PAGE,
2 * MAIN_D,
dtype=torch.uint8 if fp8_cache else torch.bfloat16,
device="cuda",
).transpose(1, 2)
slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens]
slots[-1] = -1
norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2
return dict(
main_qkv=torch.randn(
num_tokens,
2 * (MAIN_HQ + MAIN_HK) * MAIN_D,
dtype=torch.bfloat16,
device="cuda",
),
main_q_norm_weight=norm_weights[0],
main_k_norm_weight=norm_weights[1],
main_eps=EPS,
main_kv_cache=cache.view(torch.float8_e4m3fn) if fp8_cache else cache,
main_slot_mapping=slots,
main_k_scale=0.5,
main_v_scale=2.0,
)
def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None:
"""Compare with fused_qk_rmsnorm_rope_gate + reshape_and_cache_flash."""
from vllm._custom_ops import reshape_and_cache_flash
from vllm.model_executor.layers.fused_qk_norm_rope import (
fused_qk_rmsnorm_rope_gate,
)
q_gate, k, v = main["main_qkv"].split(
[2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1
)
q, k, gate = fused_qk_rmsnorm_rope_gate(
q_gate,
k,
main["main_q_norm_weight"],
main["main_k_norm_weight"],
rope.cos_sin_cache,
positions,
EPS,
MAIN_HQ,
MAIN_HK,
MAIN_D,
rope.rotary_dim,
mrope_section=MROPE_SECTION if positions.ndim == 2 else None,
norm_beta=1.0,
)
kv_cache = main["main_kv_cache"]
fp8_cache = kv_cache.dtype == torch.float8_e4m3fn
cache = torch.zeros_like(kv_cache.view(torch.uint8) if fp8_cache else kv_cache)
key_cache, value_cache = cache.split(MAIN_D, dim=-1)
reshape_and_cache_flash(
k.view(-1, MAIN_HK, MAIN_D),
v.view(-1, MAIN_HK, MAIN_D),
key_cache,
value_cache,
main["main_slot_mapping"],
"fp8" if fp8_cache else "auto",
torch.tensor(main["main_k_scale"], device="cuda"),
torch.tensor(main["main_v_scale"], device="cuda"),
)
torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL)
assert torch.equal(gate_out.flatten(1), gate)
if fp8_cache:
assert_fp8_within_one_ulp(kv_cache, cache.view(torch.float8_e4m3fn))
else:
torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL)
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
@@ -86,7 +163,7 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
],
)
def test_qsa_fused_pre_indexer_matches_unfused(
def test_qsa_fused_prepare_matches_unfused(
indexer_dtype,
mrope,
is_2d_positions,
@@ -246,7 +323,9 @@ def test_qsa_fused_pre_indexer_matches_unfused(
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
qsa_pre_indexer(
# The main-attention cache follows the recipe's pairing with the indexer.
main = _make_main_inputs(num_tokens, indexer_dtype == torch.float8_e4m3fn)
main_q_out, main_gate_out = qsa_prepare(
projected_qk[:, : HQ * D],
projected_qk[:, HQ * D :],
positions,
@@ -266,7 +345,9 @@ def test_qsa_fused_pre_indexer_matches_unfused(
compress_ratio=CR,
mrope_section=MROPE_SECTION if mrope else None,
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
**main,
)
_check_main_outputs(main, main_q_out, main_gate_out, rope, positions)
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
unfused_query = gemma_rmsnorm(
+15 -4
View File
@@ -56,16 +56,24 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
rope_position_offset=0,
),
compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)),
use_fused_pre_indexer=True,
rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)),
q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
k_layernorm=SimpleNamespace(weight=torch.ones(1)),
compress_ratio=2,
)
attn = SimpleNamespace(
use_fused_qsa_prepare=True,
kv_cache=torch.empty(0, 1, 1, 2),
kv_cache_dtype="auto",
q_norm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
k_norm=SimpleNamespace(weight=torch.ones(1)),
_k_scale_float=1.0,
_v_scale_float=1.0,
)
monkeypatch.setattr(
indexer_qsa,
"qsa_pre_indexer",
"qsa_prepare",
lambda *args, **kwargs: updates.append((args, kwargs)),
)
monkeypatch.setattr(
@@ -79,11 +87,14 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
lambda *args, **kwargs: selections.append((args, kwargs)),
)
actual = indexer_qsa.QSAIndexer.forward(
actual, _ = indexer_qsa.QSAIndexer.forward(
indexer,
torch.zeros(2, 2),
torch.tensor([7, 8]),
rows,
attn=attn,
qkv=torch.zeros(2, 4),
slot_mapping=torch.arange(2),
)
assert actual is rows
@@ -512,7 +523,6 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
_metadata=lambda: (raw_metadata, compressed_metadata),
skip_topk=True,
index_kv_heads=1,
use_fused_pre_indexer=False,
index_n_heads=1,
index_head_dim=64,
indexer_dtype=torch.bfloat16,
@@ -535,6 +545,7 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1),
torch.zeros(8, dtype=torch.long, device=device),
torch.full((5, 5), -1, dtype=torch.int32, device=device),
attn=SimpleNamespace(use_fused_qsa_prepare=False),
)
torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]])
expected_compressed = torch.zeros_like(compressed_cache)
+36 -10
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Qwen4Exp weight-free QSA indexer."""
from typing import cast
from typing import TYPE_CHECKING, cast
import torch
from torch import nn
@@ -23,7 +23,10 @@ from ..common.qsa_cache import (
QSAKeyStateCache,
canonical_qsa_rope_positions,
)
from .ops.qsa_pre_indexer import qsa_pre_indexer
from .ops.qsa_prepare import qsa_prepare
if TYPE_CHECKING:
from .qsa import Qwen4ExpQSAAttention
def apply_qsa_rope(
@@ -235,7 +238,11 @@ class QSAIndexer(nn.Module):
projected_qk: torch.Tensor,
positions: torch.Tensor,
out: torch.Tensor | None = None,
) -> torch.Tensor:
*,
attn: "Qwen4ExpQSAAttention",
qkv: torch.Tensor | None = None,
slot_mapping: torch.Tensor | None = None,
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]:
"""Update side caches and select token indices from pre-projected Q/K.
Returns the packed buffer of shape [num_tokens, output_width + 1]:
@@ -243,12 +250,17 @@ class QSAIndexer(nn.Module):
request-relative token indices, and the trailing column is the row's
valid-entry count (the attention kernel's loop bound, never a token
index).
With ``attn.use_fused_qsa_prepare``, the same launch also writes
``attn``'s K/V into ``attn.kv_cache`` at ``slot_mapping`` and prepares
its Q and gate from ``qkv``, returned as the second element (None
otherwise). ``qkv`` and ``slot_mapping`` are only read in that mode.
"""
metadata = self._metadata()
if metadata is None:
# Preserve step-0 indices when later MTP steps reuse the buffer.
if self.skip_topk and out is not None:
return out
return out, None
result = torch.full(
(projected_qk.shape[0], self.packed_output_width),
-1,
@@ -259,8 +271,8 @@ class QSAIndexer(nn.Module):
result[:, -1] = 0
if out is not None:
out.copy_(result)
return out
return result
return out, None
return result, None
from .ops.qsa import qsa_compress_groups_with_ratio, qsa_store_cache_rows
from .ops.qsa_indexer import (
@@ -284,14 +296,20 @@ class QSAIndexer(nn.Module):
raw_key_state_cache = self.raw_key_cache
compressed_key_cache = self.compressed_key_cache.kv_cache
if self.use_fused_pre_indexer:
main_outputs: tuple[torch.Tensor, torch.Tensor] | None = None
if attn.use_fused_qsa_prepare:
if qkv is None or slot_mapping is None:
raise ValueError("fused QSA prepare requires qkv and slot_mapping")
q = projected_q.new_empty(
num_tokens,
self.index_n_heads,
self.index_head_dim,
dtype=self.indexer_dtype,
)
qsa_pre_indexer(
main_kv_cache = attn.kv_cache.transpose(1, 2)
if attn.kv_cache_dtype in ("fp8", "fp8_e4m3"):
main_kv_cache = main_kv_cache.view(torch.float8_e4m3fn)
main_outputs = qsa_prepare(
projected_q,
raw_keys,
positions,
@@ -315,6 +333,14 @@ class QSAIndexer(nn.Module):
if raw_key_state_cache.rope_position_cache is not None
else None
),
main_qkv=qkv[:num_tokens],
main_q_norm_weight=attn.q_norm.weight,
main_k_norm_weight=attn.k_norm.weight,
main_eps=attn.q_norm.variance_epsilon,
main_kv_cache=main_kv_cache,
main_slot_mapping=slot_mapping[:num_tokens],
main_k_scale=attn._k_scale_float,
main_v_scale=attn._v_scale_float,
)
else:
# Unfused reference path
@@ -385,7 +411,7 @@ class QSAIndexer(nn.Module):
if self.skip_topk:
if out is None:
raise RuntimeError("QSA top-k reuse requires an output buffer")
return out
return out, main_outputs
if out is None:
out = torch.empty(
@@ -448,7 +474,7 @@ class QSAIndexer(nn.Module):
self.token_topk,
out,
)
return out
return out, main_outputs
__all__ = ["QSAIndexer", "apply_qsa_rope"]
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused QSA pre-indexer kernel for Qwen4Exp."""
"""Fused QSA prepare kernel for Qwen4Exp."""
import torch
@@ -80,6 +80,27 @@ def _norm_rope(
return tl.reshape(result, (TILE_T, TILE_H, D))
@triton.jit
def _to_dst_dtype(x, dst, scale):
"""Round to BF16 like the unfused path, then scale for an FP8 destination."""
out_ty = dst.dtype.element_ty
x = x.to(tl.bfloat16)
if out_ty == tl.float8e4nv:
x = x.to(tl.float32) / scale
return x.to(out_ty)
@triton.jit
def _store_rotated(dst, y, o1, o2, scale):
"""Store a normalized head whose first ``2 * len(o1)`` dims are rotated."""
HALF: tl.constexpr = o1.shape[0]
dims = tl.arange(0, y.shape[0])
rot = tl.arange(0, HALF)
tl.store(dst + dims, _to_dst_dtype(y, dst, scale), mask=dims >= 2 * HALF)
tl.store(dst + rot, _to_dst_dtype(o1, dst, scale))
tl.store(dst + HALF + rot, _to_dst_dtype(o2, dst, scale))
@triton.jit(
do_not_specialize=[
"num_tokens",
@@ -88,7 +109,7 @@ def _norm_rope(
"num_k_work",
]
)
def _qsa_pre_indexer_kernel(
def _qsa_prepare_kernel(
q_ptr,
q_stride_token,
k_ptr,
@@ -132,8 +153,86 @@ def _qsa_pre_indexer_kernel(
CACHE_HAS_ROPE_POS: tl.constexpr,
MROPE_H: tl.constexpr,
MROPE_W: tl.constexpr,
main_qkv_ptr,
main_q_norm_weight_ptr,
main_k_norm_weight_ptr,
main_eps,
main_q_out_ptr,
main_gate_out_ptr,
main_cache_ptr,
main_cache_stride_block,
main_cache_stride_token,
main_cache_stride_head,
main_slots_ptr,
main_k_scale,
main_v_scale,
MAIN_HQ: tl.constexpr,
MAIN_HK: tl.constexpr,
MAIN_D: tl.constexpr,
MAIN_PAGE_SIZE: tl.constexpr,
):
pid = tl.program_id(0)
num_index_work = num_k_work + tl.cdiv(num_tokens, TILE_T_Q) * tl.cdiv(HQ, TILE_H_Q)
if pid >= num_index_work:
# Main attention: one program per (token, Q or KV head) after the
# indexer work. RoPE covers the first D // 2 dims, the width of the
# shared cos/sin table.
main_pid = pid - num_index_work
token = main_pid // (MAIN_HQ + MAIN_HK)
head = main_pid % (MAIN_HQ + MAIN_HK)
HALF: tl.constexpr = D // 4
dims = tl.arange(0, MAIN_D)
rot = tl.arange(0, HALF)
row = main_qkv_ptr + token * (2 * (MAIN_HQ + MAIN_HK) * MAIN_D)
is_k = head >= MAIN_HQ
kv_head = head - MAIN_HQ
if is_k:
src = row + (2 * MAIN_HQ + kv_head) * MAIN_D
weight_ptr = main_k_norm_weight_ptr
else:
src = row + 2 * head * MAIN_D
weight_ptr = main_q_norm_weight_ptr
x = tl.load(src + dims).to(tl.float32)
inv_rms = tl.rsqrt(tl.sum(x * x, axis=0) / MAIN_D + main_eps)
w = tl.load(weight_ptr + dims).to(tl.float32) + 1.0
y = x * inv_rms * w
x1 = tl.load(src + rot).to(tl.float32)
x2 = tl.load(src + HALF + rot).to(tl.float32)
w1 = tl.load(weight_ptr + rot).to(tl.float32) + 1.0
w2 = tl.load(weight_ptr + HALF + rot).to(tl.float32) + 1.0
x1 = (x1 * inv_rms * w1).to(tl.bfloat16).to(tl.float32)
x2 = (x2 * inv_rms * w2).to(tl.bfloat16).to(tl.float32)
pos = tl.load(pos_ptr + token * pos_stride_token).to(tl.int64)
if IS_2D_POSITIONS:
pos_h = tl.load(pos_ptr + pos_stride_axis + token * pos_stride_token)
pos_w = tl.load(pos_ptr + 2 * pos_stride_axis + token * pos_stride_token)
is_h = (rot % 3 == 1) & (rot < 3 * MROPE_H)
is_w = (rot % 3 == 2) & (rot < 3 * MROPE_W)
pos = tl.where(
is_h, pos_h.to(tl.int64), tl.where(is_w, pos_w.to(tl.int64), pos)
)
cos = tl.load(cos_sin_ptr + pos * (D // 2) + rot).to(tl.float32)
sin = tl.load(cos_sin_ptr + pos * (D // 2) + HALF + rot).to(tl.float32)
o1 = x1 * cos - x2 * sin
o2 = x2 * cos + x1 * sin
if is_k:
slot = tl.load(main_slots_ptr + token).to(tl.int64)
if slot >= 0:
dst = (
main_cache_ptr
+ (slot // MAIN_PAGE_SIZE) * main_cache_stride_block
+ (slot % MAIN_PAGE_SIZE) * main_cache_stride_token
+ kv_head * main_cache_stride_head
)
_store_rotated(dst, y, o1, o2, main_k_scale)
v = tl.load(src + MAIN_HK * MAIN_D + dims)
tl.store(dst + MAIN_D + dims, _to_dst_dtype(v, dst, main_v_scale))
else:
out = (token * MAIN_HQ + head) * MAIN_D
_store_rotated(main_q_out_ptr + out, y, o1, o2, None)
gate = tl.load(src + MAIN_D + dims)
tl.store(main_gate_out_ptr + out + dims, gate)
return
# K work occupies the first programs; the remaining programs tile Q. This
# keeps both paths in one launch while leaving their register shapes
# independent.
@@ -400,7 +499,7 @@ def _qsa_pre_indexer_kernel(
tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot)
def qsa_pre_indexer(
def qsa_prepare(
q: torch.Tensor,
k: torch.Tensor,
positions: torch.Tensor,
@@ -421,11 +520,28 @@ def qsa_pre_indexer(
compress_ratio: int,
mrope_section: tuple[int, int, int] | None,
rope_pos_offset: int | None,
) -> None:
"""Normalize Q, compress K, then update the circular raw state."""
main_qkv: torch.Tensor,
main_q_norm_weight: torch.Tensor,
main_k_norm_weight: torch.Tensor,
main_eps: float,
main_kv_cache: torch.Tensor,
main_slot_mapping: torch.Tensor,
main_k_scale: float,
main_v_scale: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Normalize Q, compress K, then update the circular raw state.
Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write)
and returns its Q and gate.
"""
num_tokens = q.shape[0]
main_head_dim = main_kv_cache.shape[-1] // 2
num_main_kv_heads = main_kv_cache.shape[2]
num_main_q_heads = main_qkv.shape[1] // (2 * main_head_dim) - num_main_kv_heads
main_q_out = main_qkv.new_empty(num_tokens, num_main_q_heads, main_head_dim)
main_gate_out = torch.empty_like(main_q_out)
if num_tokens == 0:
return
return main_q_out, main_gate_out
num_q_heads, head_dim = q_out.shape[1:]
assert cos_sin_cache.shape[-1] * 2 == head_dim
assert q.shape == (num_tokens, num_q_heads * head_dim)
@@ -450,6 +566,9 @@ def qsa_pre_indexer(
pos_stride_axis, pos_stride_token = 0, positions.stride(0)
section = mrope_section if mrope_section is not None else (0, 0, 0)
assert len(section) == 3
qkv_width = 2 * (num_main_q_heads + num_main_kv_heads) * main_head_dim
assert main_qkv.shape == (num_tokens, qkv_width) and main_qkv.is_contiguous()
assert main_slot_mapping.shape == (num_tokens,)
if num_tokens <= 4096:
TILE_T_Q, TILE_H_Q = 2, 2
@@ -457,7 +576,8 @@ def qsa_pre_indexer(
TILE_T_Q, TILE_H_Q = 2, 4
num_k_work = k_work_metadata.shape[0]
num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q)
_qsa_pre_indexer_kernel[(num_k_work + num_q_work,)](
num_main_work = num_tokens * (num_main_q_heads + num_main_kv_heads)
_qsa_prepare_kernel[(num_k_work + num_q_work + num_main_work,)](
q,
q.stride(0),
k,
@@ -501,8 +621,26 @@ def qsa_pre_indexer(
CACHE_HAS_ROPE_POS=cache_has_rope_pos,
MROPE_H=section[1],
MROPE_W=section[2],
main_qkv_ptr=main_qkv,
main_q_norm_weight_ptr=main_q_norm_weight,
main_k_norm_weight_ptr=main_k_norm_weight,
main_eps=main_eps,
main_q_out_ptr=main_q_out,
main_gate_out_ptr=main_gate_out,
main_cache_ptr=main_kv_cache,
main_cache_stride_block=main_kv_cache.stride(0),
main_cache_stride_token=main_kv_cache.stride(1),
main_cache_stride_head=main_kv_cache.stride(2),
main_slots_ptr=main_slot_mapping,
main_k_scale=main_k_scale,
main_v_scale=main_v_scale,
MAIN_HQ=num_main_q_heads,
MAIN_HK=num_main_kv_heads,
MAIN_D=main_head_dim,
MAIN_PAGE_SIZE=main_kv_cache.shape[1],
num_warps=1,
)
return main_q_out, main_gate_out
__all__ = ["qsa_pre_indexer"]
__all__ = ["qsa_prepare"]
+40 -18
View File
@@ -407,6 +407,12 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
quant_config=quant_config,
prefix=f"{prefix}.indexer",
)
# One launch does the indexer prepare, the main QK-norm/RoPE/gate and
# the main K/V cache write (see QSAIndexer.forward); otherwise all of
# them take the separate kernels.
self.use_fused_qsa_prepare = (
self.use_fused_qk_norm_rope_gate and self.indexer.use_fused_pre_indexer
)
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
# PACKED selection buffer: the trailing column holds each row's
# valid-entry count (written by the expand kernel) — never a token
@@ -446,12 +452,15 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
self,
projected_qk: torch.Tensor,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
query: torch.Tensor | None,
key: torch.Tensor | None,
value: torch.Tensor | None,
output: torch.Tensor,
output_gate: torch.Tensor,
output_gate: torch.Tensor | None,
qkv: torch.Tensor,
) -> None:
# query/key/value/output_gate are None when the fused prepare runs
# inside the indexer launch.
metadata = get_forward_context().attn_metadata
if isinstance(metadata, list):
metadata = metadata[0]
@@ -469,21 +478,29 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
)
if side_metadata.num_actual_tokens != num_tokens:
raise RuntimeError("QSA main and side metadata token counts disagree")
selected = self.indexer(
selected, main_outputs = self.indexer(
projected_qk,
positions,
self.topk_indices_buffer[:num_tokens],
attn=self,
qkv=qkv,
slot_mapping=main_metadata.slot_mapping,
)
if selected.shape != (num_tokens, self.indexer.packed_output_width):
raise RuntimeError("QSA indexer returned an invalid selection shape")
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
impl.do_kv_cache_update(
self,
key,
value,
self.kv_cache,
main_metadata.slot_mapping,
)
if main_outputs is None:
assert key is not None and value is not None
impl.do_kv_cache_update(
self,
key,
value,
self.kv_cache,
main_metadata.slot_mapping,
)
else:
query, output_gate = main_outputs
assert query is not None and output_gate is not None
impl.forward_qsa(
self,
query,
@@ -503,13 +520,17 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v, gate = self._project_qkv_gate(qkv, positions)
assert gate is not None
num_tokens = hidden_states.shape[0]
query = q.view(num_tokens, self.num_heads, self.head_dim)
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
attn_output = torch.empty_like(query)
if not self.use_fused_qsa_prepare:
q, k, v, gate = self._project_qkv_gate(qkv, positions)
assert gate is not None
query = q.view(num_tokens, self.num_heads, self.head_dim)
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
else:
# Norm/RoPE/gate and the K/V cache write happen inside _run_qsa.
query = key = value = gate = None
attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim)
# Keep the index projection outside the eager break.
projected_qk, _ = self.indexer.index_qk_proj(hidden_states)
self._run_qsa(
@@ -520,6 +541,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
value,
attn_output,
gate,
qkv,
)
flat_output = attn_output.view(num_tokens, -1)
output, _ = self.o_proj(flat_output)