From c9578f10ab3eaf252742960db3fd08f11cf6d25a Mon Sep 17 00:00:00 2001 From: Shuolei Wang <46160365+ShuoleiWang@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:21:06 +0800 Subject: [PATCH] [Qwen3.8-Flash-Next] Fuse main QK-norm/RoPE/gate and KV-cache write into the QSA pre-indexer launch (#57097) Signed-off-by: Shuolei Wang Signed-off-by: Thien Co-authored-by: Thien --- .buildkite/test_areas/models_basic.yaml | 2 +- ...qsa_pre_indexer.py => test_qsa_prepare.py} | 93 ++++++++++- tests/models/qwen4_exp/test_qsa_reference.py | 19 ++- vllm/models/qwen4_exp/nvidia/indexer_qsa.py | 46 ++++-- .../{qsa_pre_indexer.py => qsa_prepare.py} | 154 +++++++++++++++++- vllm/models/qwen4_exp/nvidia/qsa.py | 58 +++++-- 6 files changed, 325 insertions(+), 47 deletions(-) rename tests/models/qwen4_exp/{test_qsa_pre_indexer.py => test_qsa_prepare.py} (77%) rename vllm/models/qwen4_exp/nvidia/ops/{qsa_pre_indexer.py => qsa_prepare.py} (75%) diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 7c03db5cb4..b817b9a1fb 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -178,7 +178,7 @@ steps: - vllm/v1/attention/backends/short_conv_attn.py - tests/models/qwen4_exp/ commands: - - pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_pre_indexer.py models/qwen4_exp/test_qsa_reference.py + - pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_prepare.py models/qwen4_exp/test_qsa_reference.py mirror: amd: label: ":amd: (MI355 DPX) Qwen4 Exp" diff --git a/tests/models/qwen4_exp/test_qsa_pre_indexer.py b/tests/models/qwen4_exp/test_qsa_prepare.py similarity index 77% rename from tests/models/qwen4_exp/test_qsa_pre_indexer.py rename to tests/models/qwen4_exp/test_qsa_prepare.py index 8d4df3dbc9..14b836c132 100644 --- a/tests/models/qwen4_exp/test_qsa_pre_indexer.py +++ b/tests/models/qwen4_exp/test_qsa_prepare.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Correctness tests for the fused QSA pre-indexer.""" +"""Correctness tests for the fused QSA prepare kernel.""" import pytest import torch @@ -15,9 +15,7 @@ from vllm.models.qwen4_exp.nvidia.ops.qsa import ( qsa_compress_groups_with_ratio, qsa_store_cache_rows, ) -from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import ( - qsa_pre_indexer, -) +from vllm.models.qwen4_exp.nvidia.ops.qsa_prepare import qsa_prepare from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON @@ -36,6 +34,7 @@ ROPE_POS_OFFSET = D RTOL = 1.6e-2 ATOL = 1e-2 MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8]) +MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16 def _make_block_table(block_counts): @@ -61,6 +60,84 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all()) +def _make_main_inputs(num_tokens: int, fp8_cache: bool) -> dict: + """Random main-attention arguments with a paged cache in vLLM's layout.""" + num_blocks = num_tokens // MAIN_PAGE + 2 + cache = torch.zeros( + num_blocks, + MAIN_HK, + MAIN_PAGE, + 2 * MAIN_D, + dtype=torch.uint8 if fp8_cache else torch.bfloat16, + device="cuda", + ).transpose(1, 2) + slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens] + slots[-1] = -1 + norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2 + return dict( + main_qkv=torch.randn( + num_tokens, + 2 * (MAIN_HQ + MAIN_HK) * MAIN_D, + dtype=torch.bfloat16, + device="cuda", + ), + main_q_norm_weight=norm_weights[0], + main_k_norm_weight=norm_weights[1], + main_eps=EPS, + main_kv_cache=cache.view(torch.float8_e4m3fn) if fp8_cache else cache, + main_slot_mapping=slots, + main_k_scale=0.5, + main_v_scale=2.0, + ) + + +def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None: + """Compare with fused_qk_rmsnorm_rope_gate + reshape_and_cache_flash.""" + from vllm._custom_ops import reshape_and_cache_flash + from vllm.model_executor.layers.fused_qk_norm_rope import ( + fused_qk_rmsnorm_rope_gate, + ) + + q_gate, k, v = main["main_qkv"].split( + [2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1 + ) + q, k, gate = fused_qk_rmsnorm_rope_gate( + q_gate, + k, + main["main_q_norm_weight"], + main["main_k_norm_weight"], + rope.cos_sin_cache, + positions, + EPS, + MAIN_HQ, + MAIN_HK, + MAIN_D, + rope.rotary_dim, + mrope_section=MROPE_SECTION if positions.ndim == 2 else None, + norm_beta=1.0, + ) + kv_cache = main["main_kv_cache"] + fp8_cache = kv_cache.dtype == torch.float8_e4m3fn + cache = torch.zeros_like(kv_cache.view(torch.uint8) if fp8_cache else kv_cache) + key_cache, value_cache = cache.split(MAIN_D, dim=-1) + reshape_and_cache_flash( + k.view(-1, MAIN_HK, MAIN_D), + v.view(-1, MAIN_HK, MAIN_D), + key_cache, + value_cache, + main["main_slot_mapping"], + "fp8" if fp8_cache else "auto", + torch.tensor(main["main_k_scale"], device="cuda"), + torch.tensor(main["main_v_scale"], device="cuda"), + ) + torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL) + assert torch.equal(gate_out.flatten(1), gate) + if fp8_cache: + assert_fp8_within_one_ulp(kv_cache, cache.view(torch.float8_e4m3fn)) + else: + torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL) + + @requires_qsa_kernels @pytest.mark.usefixtures("default_vllm_config") @pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn]) @@ -86,7 +163,7 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"), ], ) -def test_qsa_fused_pre_indexer_matches_unfused( +def test_qsa_fused_prepare_matches_unfused( indexer_dtype, mrope, is_2d_positions, @@ -246,7 +323,9 @@ def test_qsa_fused_pre_indexer_matches_unfused( k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2 fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device) - qsa_pre_indexer( + # The main-attention cache follows the recipe's pairing with the indexer. + main = _make_main_inputs(num_tokens, indexer_dtype == torch.float8_e4m3fn) + main_q_out, main_gate_out = qsa_prepare( projected_qk[:, : HQ * D], projected_qk[:, HQ * D :], positions, @@ -266,7 +345,9 @@ def test_qsa_fused_pre_indexer_matches_unfused( compress_ratio=CR, mrope_section=MROPE_SECTION if mrope else None, rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None, + **main, ) + _check_main_outputs(main, main_q_out, main_gate_out, rope, positions) unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D) unfused_query = gemma_rmsnorm( diff --git a/tests/models/qwen4_exp/test_qsa_reference.py b/tests/models/qwen4_exp/test_qsa_reference.py index a6f3b6557d..dc431aed7a 100644 --- a/tests/models/qwen4_exp/test_qsa_reference.py +++ b/tests/models/qwen4_exp/test_qsa_reference.py @@ -56,16 +56,24 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection( rope_position_offset=0, ), compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)), - use_fused_pre_indexer=True, rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)), q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6), k_layernorm=SimpleNamespace(weight=torch.ones(1)), compress_ratio=2, ) + attn = SimpleNamespace( + use_fused_qsa_prepare=True, + kv_cache=torch.empty(0, 1, 1, 2), + kv_cache_dtype="auto", + q_norm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6), + k_norm=SimpleNamespace(weight=torch.ones(1)), + _k_scale_float=1.0, + _v_scale_float=1.0, + ) monkeypatch.setattr( indexer_qsa, - "qsa_pre_indexer", + "qsa_prepare", lambda *args, **kwargs: updates.append((args, kwargs)), ) monkeypatch.setattr( @@ -79,11 +87,14 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection( lambda *args, **kwargs: selections.append((args, kwargs)), ) - actual = indexer_qsa.QSAIndexer.forward( + actual, _ = indexer_qsa.QSAIndexer.forward( indexer, torch.zeros(2, 2), torch.tensor([7, 8]), rows, + attn=attn, + qkv=torch.zeros(2, 4), + slot_mapping=torch.arange(2), ) assert actual is rows @@ -512,7 +523,6 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None: _metadata=lambda: (raw_metadata, compressed_metadata), skip_topk=True, index_kv_heads=1, - use_fused_pre_indexer=False, index_n_heads=1, index_head_dim=64, indexer_dtype=torch.bfloat16, @@ -535,6 +545,7 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None: torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1), torch.zeros(8, dtype=torch.long, device=device), torch.full((5, 5), -1, dtype=torch.int32, device=device), + attn=SimpleNamespace(use_fused_qsa_prepare=False), ) torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]]) expected_compressed = torch.zeros_like(compressed_cache) diff --git a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py index f4b968f152..14137de695 100644 --- a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py +++ b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Qwen4Exp weight-free QSA indexer.""" -from typing import cast +from typing import TYPE_CHECKING, cast import torch from torch import nn @@ -23,7 +23,10 @@ from ..common.qsa_cache import ( QSAKeyStateCache, canonical_qsa_rope_positions, ) -from .ops.qsa_pre_indexer import qsa_pre_indexer +from .ops.qsa_prepare import qsa_prepare + +if TYPE_CHECKING: + from .qsa import Qwen4ExpQSAAttention def apply_qsa_rope( @@ -235,7 +238,11 @@ class QSAIndexer(nn.Module): projected_qk: torch.Tensor, positions: torch.Tensor, out: torch.Tensor | None = None, - ) -> torch.Tensor: + *, + attn: "Qwen4ExpQSAAttention", + qkv: torch.Tensor | None = None, + slot_mapping: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]: """Update side caches and select token indices from pre-projected Q/K. Returns the packed buffer of shape [num_tokens, output_width + 1]: @@ -243,12 +250,17 @@ class QSAIndexer(nn.Module): request-relative token indices, and the trailing column is the row's valid-entry count (the attention kernel's loop bound, never a token index). + + With ``attn.use_fused_qsa_prepare``, the same launch also writes + ``attn``'s K/V into ``attn.kv_cache`` at ``slot_mapping`` and prepares + its Q and gate from ``qkv``, returned as the second element (None + otherwise). ``qkv`` and ``slot_mapping`` are only read in that mode. """ metadata = self._metadata() if metadata is None: # Preserve step-0 indices when later MTP steps reuse the buffer. if self.skip_topk and out is not None: - return out + return out, None result = torch.full( (projected_qk.shape[0], self.packed_output_width), -1, @@ -259,8 +271,8 @@ class QSAIndexer(nn.Module): result[:, -1] = 0 if out is not None: out.copy_(result) - return out - return result + return out, None + return result, None from .ops.qsa import qsa_compress_groups_with_ratio, qsa_store_cache_rows from .ops.qsa_indexer import ( @@ -284,14 +296,20 @@ class QSAIndexer(nn.Module): raw_key_state_cache = self.raw_key_cache compressed_key_cache = self.compressed_key_cache.kv_cache - if self.use_fused_pre_indexer: + main_outputs: tuple[torch.Tensor, torch.Tensor] | None = None + if attn.use_fused_qsa_prepare: + if qkv is None or slot_mapping is None: + raise ValueError("fused QSA prepare requires qkv and slot_mapping") q = projected_q.new_empty( num_tokens, self.index_n_heads, self.index_head_dim, dtype=self.indexer_dtype, ) - qsa_pre_indexer( + main_kv_cache = attn.kv_cache.transpose(1, 2) + if attn.kv_cache_dtype in ("fp8", "fp8_e4m3"): + main_kv_cache = main_kv_cache.view(torch.float8_e4m3fn) + main_outputs = qsa_prepare( projected_q, raw_keys, positions, @@ -315,6 +333,14 @@ class QSAIndexer(nn.Module): if raw_key_state_cache.rope_position_cache is not None else None ), + main_qkv=qkv[:num_tokens], + main_q_norm_weight=attn.q_norm.weight, + main_k_norm_weight=attn.k_norm.weight, + main_eps=attn.q_norm.variance_epsilon, + main_kv_cache=main_kv_cache, + main_slot_mapping=slot_mapping[:num_tokens], + main_k_scale=attn._k_scale_float, + main_v_scale=attn._v_scale_float, ) else: # Unfused reference path @@ -385,7 +411,7 @@ class QSAIndexer(nn.Module): if self.skip_topk: if out is None: raise RuntimeError("QSA top-k reuse requires an output buffer") - return out + return out, main_outputs if out is None: out = torch.empty( @@ -448,7 +474,7 @@ class QSAIndexer(nn.Module): self.token_topk, out, ) - return out + return out, main_outputs __all__ = ["QSAIndexer", "apply_qsa_rope"] diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py b/vllm/models/qwen4_exp/nvidia/ops/qsa_prepare.py similarity index 75% rename from vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py rename to vllm/models/qwen4_exp/nvidia/ops/qsa_prepare.py index d996f3977a..6e97a89781 100644 --- a/vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa_prepare.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Fused QSA pre-indexer kernel for Qwen4Exp.""" +"""Fused QSA prepare kernel for Qwen4Exp.""" import torch @@ -80,6 +80,27 @@ def _norm_rope( return tl.reshape(result, (TILE_T, TILE_H, D)) +@triton.jit +def _to_dst_dtype(x, dst, scale): + """Round to BF16 like the unfused path, then scale for an FP8 destination.""" + out_ty = dst.dtype.element_ty + x = x.to(tl.bfloat16) + if out_ty == tl.float8e4nv: + x = x.to(tl.float32) / scale + return x.to(out_ty) + + +@triton.jit +def _store_rotated(dst, y, o1, o2, scale): + """Store a normalized head whose first ``2 * len(o1)`` dims are rotated.""" + HALF: tl.constexpr = o1.shape[0] + dims = tl.arange(0, y.shape[0]) + rot = tl.arange(0, HALF) + tl.store(dst + dims, _to_dst_dtype(y, dst, scale), mask=dims >= 2 * HALF) + tl.store(dst + rot, _to_dst_dtype(o1, dst, scale)) + tl.store(dst + HALF + rot, _to_dst_dtype(o2, dst, scale)) + + @triton.jit( do_not_specialize=[ "num_tokens", @@ -88,7 +109,7 @@ def _norm_rope( "num_k_work", ] ) -def _qsa_pre_indexer_kernel( +def _qsa_prepare_kernel( q_ptr, q_stride_token, k_ptr, @@ -132,8 +153,86 @@ def _qsa_pre_indexer_kernel( CACHE_HAS_ROPE_POS: tl.constexpr, MROPE_H: tl.constexpr, MROPE_W: tl.constexpr, + main_qkv_ptr, + main_q_norm_weight_ptr, + main_k_norm_weight_ptr, + main_eps, + main_q_out_ptr, + main_gate_out_ptr, + main_cache_ptr, + main_cache_stride_block, + main_cache_stride_token, + main_cache_stride_head, + main_slots_ptr, + main_k_scale, + main_v_scale, + MAIN_HQ: tl.constexpr, + MAIN_HK: tl.constexpr, + MAIN_D: tl.constexpr, + MAIN_PAGE_SIZE: tl.constexpr, ): pid = tl.program_id(0) + num_index_work = num_k_work + tl.cdiv(num_tokens, TILE_T_Q) * tl.cdiv(HQ, TILE_H_Q) + if pid >= num_index_work: + # Main attention: one program per (token, Q or KV head) after the + # indexer work. RoPE covers the first D // 2 dims, the width of the + # shared cos/sin table. + main_pid = pid - num_index_work + token = main_pid // (MAIN_HQ + MAIN_HK) + head = main_pid % (MAIN_HQ + MAIN_HK) + HALF: tl.constexpr = D // 4 + dims = tl.arange(0, MAIN_D) + rot = tl.arange(0, HALF) + row = main_qkv_ptr + token * (2 * (MAIN_HQ + MAIN_HK) * MAIN_D) + is_k = head >= MAIN_HQ + kv_head = head - MAIN_HQ + if is_k: + src = row + (2 * MAIN_HQ + kv_head) * MAIN_D + weight_ptr = main_k_norm_weight_ptr + else: + src = row + 2 * head * MAIN_D + weight_ptr = main_q_norm_weight_ptr + x = tl.load(src + dims).to(tl.float32) + inv_rms = tl.rsqrt(tl.sum(x * x, axis=0) / MAIN_D + main_eps) + w = tl.load(weight_ptr + dims).to(tl.float32) + 1.0 + y = x * inv_rms * w + x1 = tl.load(src + rot).to(tl.float32) + x2 = tl.load(src + HALF + rot).to(tl.float32) + w1 = tl.load(weight_ptr + rot).to(tl.float32) + 1.0 + w2 = tl.load(weight_ptr + HALF + rot).to(tl.float32) + 1.0 + x1 = (x1 * inv_rms * w1).to(tl.bfloat16).to(tl.float32) + x2 = (x2 * inv_rms * w2).to(tl.bfloat16).to(tl.float32) + pos = tl.load(pos_ptr + token * pos_stride_token).to(tl.int64) + if IS_2D_POSITIONS: + pos_h = tl.load(pos_ptr + pos_stride_axis + token * pos_stride_token) + pos_w = tl.load(pos_ptr + 2 * pos_stride_axis + token * pos_stride_token) + is_h = (rot % 3 == 1) & (rot < 3 * MROPE_H) + is_w = (rot % 3 == 2) & (rot < 3 * MROPE_W) + pos = tl.where( + is_h, pos_h.to(tl.int64), tl.where(is_w, pos_w.to(tl.int64), pos) + ) + cos = tl.load(cos_sin_ptr + pos * (D // 2) + rot).to(tl.float32) + sin = tl.load(cos_sin_ptr + pos * (D // 2) + HALF + rot).to(tl.float32) + o1 = x1 * cos - x2 * sin + o2 = x2 * cos + x1 * sin + if is_k: + slot = tl.load(main_slots_ptr + token).to(tl.int64) + if slot >= 0: + dst = ( + main_cache_ptr + + (slot // MAIN_PAGE_SIZE) * main_cache_stride_block + + (slot % MAIN_PAGE_SIZE) * main_cache_stride_token + + kv_head * main_cache_stride_head + ) + _store_rotated(dst, y, o1, o2, main_k_scale) + v = tl.load(src + MAIN_HK * MAIN_D + dims) + tl.store(dst + MAIN_D + dims, _to_dst_dtype(v, dst, main_v_scale)) + else: + out = (token * MAIN_HQ + head) * MAIN_D + _store_rotated(main_q_out_ptr + out, y, o1, o2, None) + gate = tl.load(src + MAIN_D + dims) + tl.store(main_gate_out_ptr + out + dims, gate) + return # K work occupies the first programs; the remaining programs tile Q. This # keeps both paths in one launch while leaving their register shapes # independent. @@ -400,7 +499,7 @@ def _qsa_pre_indexer_kernel( tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot) -def qsa_pre_indexer( +def qsa_prepare( q: torch.Tensor, k: torch.Tensor, positions: torch.Tensor, @@ -421,11 +520,28 @@ def qsa_pre_indexer( compress_ratio: int, mrope_section: tuple[int, int, int] | None, rope_pos_offset: int | None, -) -> None: - """Normalize Q, compress K, then update the circular raw state.""" + main_qkv: torch.Tensor, + main_q_norm_weight: torch.Tensor, + main_k_norm_weight: torch.Tensor, + main_eps: float, + main_kv_cache: torch.Tensor, + main_slot_mapping: torch.Tensor, + main_k_scale: float, + main_v_scale: float, +) -> tuple[torch.Tensor, torch.Tensor]: + """Normalize Q, compress K, then update the circular raw state. + + Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write) + and returns its Q and gate. + """ num_tokens = q.shape[0] + main_head_dim = main_kv_cache.shape[-1] // 2 + num_main_kv_heads = main_kv_cache.shape[2] + num_main_q_heads = main_qkv.shape[1] // (2 * main_head_dim) - num_main_kv_heads + main_q_out = main_qkv.new_empty(num_tokens, num_main_q_heads, main_head_dim) + main_gate_out = torch.empty_like(main_q_out) if num_tokens == 0: - return + return main_q_out, main_gate_out num_q_heads, head_dim = q_out.shape[1:] assert cos_sin_cache.shape[-1] * 2 == head_dim assert q.shape == (num_tokens, num_q_heads * head_dim) @@ -450,6 +566,9 @@ def qsa_pre_indexer( pos_stride_axis, pos_stride_token = 0, positions.stride(0) section = mrope_section if mrope_section is not None else (0, 0, 0) assert len(section) == 3 + qkv_width = 2 * (num_main_q_heads + num_main_kv_heads) * main_head_dim + assert main_qkv.shape == (num_tokens, qkv_width) and main_qkv.is_contiguous() + assert main_slot_mapping.shape == (num_tokens,) if num_tokens <= 4096: TILE_T_Q, TILE_H_Q = 2, 2 @@ -457,7 +576,8 @@ def qsa_pre_indexer( TILE_T_Q, TILE_H_Q = 2, 4 num_k_work = k_work_metadata.shape[0] num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q) - _qsa_pre_indexer_kernel[(num_k_work + num_q_work,)]( + num_main_work = num_tokens * (num_main_q_heads + num_main_kv_heads) + _qsa_prepare_kernel[(num_k_work + num_q_work + num_main_work,)]( q, q.stride(0), k, @@ -501,8 +621,26 @@ def qsa_pre_indexer( CACHE_HAS_ROPE_POS=cache_has_rope_pos, MROPE_H=section[1], MROPE_W=section[2], + main_qkv_ptr=main_qkv, + main_q_norm_weight_ptr=main_q_norm_weight, + main_k_norm_weight_ptr=main_k_norm_weight, + main_eps=main_eps, + main_q_out_ptr=main_q_out, + main_gate_out_ptr=main_gate_out, + main_cache_ptr=main_kv_cache, + main_cache_stride_block=main_kv_cache.stride(0), + main_cache_stride_token=main_kv_cache.stride(1), + main_cache_stride_head=main_kv_cache.stride(2), + main_slots_ptr=main_slot_mapping, + main_k_scale=main_k_scale, + main_v_scale=main_v_scale, + MAIN_HQ=num_main_q_heads, + MAIN_HK=num_main_kv_heads, + MAIN_D=main_head_dim, + MAIN_PAGE_SIZE=main_kv_cache.shape[1], num_warps=1, ) + return main_q_out, main_gate_out -__all__ = ["qsa_pre_indexer"] +__all__ = ["qsa_prepare"] diff --git a/vllm/models/qwen4_exp/nvidia/qsa.py b/vllm/models/qwen4_exp/nvidia/qsa.py index b6da89a863..62aa6459fb 100644 --- a/vllm/models/qwen4_exp/nvidia/qsa.py +++ b/vllm/models/qwen4_exp/nvidia/qsa.py @@ -407,6 +407,12 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): quant_config=quant_config, prefix=f"{prefix}.indexer", ) + # One launch does the indexer prepare, the main QK-norm/RoPE/gate and + # the main K/V cache write (see QSAIndexer.forward); otherwise all of + # them take the separate kernels. + self.use_fused_qsa_prepare = ( + self.use_fused_qk_norm_rope_gate and self.indexer.use_fused_pre_indexer + ) max_tokens = vllm_config.scheduler_config.max_num_batched_tokens # PACKED selection buffer: the trailing column holds each row's # valid-entry count (written by the expand kernel) — never a token @@ -446,12 +452,15 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): self, projected_qk: torch.Tensor, positions: torch.Tensor, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, + query: torch.Tensor | None, + key: torch.Tensor | None, + value: torch.Tensor | None, output: torch.Tensor, - output_gate: torch.Tensor, + output_gate: torch.Tensor | None, + qkv: torch.Tensor, ) -> None: + # query/key/value/output_gate are None when the fused prepare runs + # inside the indexer launch. metadata = get_forward_context().attn_metadata if isinstance(metadata, list): metadata = metadata[0] @@ -469,21 +478,29 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): ) if side_metadata.num_actual_tokens != num_tokens: raise RuntimeError("QSA main and side metadata token counts disagree") - selected = self.indexer( + selected, main_outputs = self.indexer( projected_qk, positions, self.topk_indices_buffer[:num_tokens], + attn=self, + qkv=qkv, + slot_mapping=main_metadata.slot_mapping, ) if selected.shape != (num_tokens, self.indexer.packed_output_width): raise RuntimeError("QSA indexer returned an invalid selection shape") impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl) - impl.do_kv_cache_update( - self, - key, - value, - self.kv_cache, - main_metadata.slot_mapping, - ) + if main_outputs is None: + assert key is not None and value is not None + impl.do_kv_cache_update( + self, + key, + value, + self.kv_cache, + main_metadata.slot_mapping, + ) + else: + query, output_gate = main_outputs + assert query is not None and output_gate is not None impl.forward_qsa( self, query, @@ -503,13 +520,17 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): hidden_states: torch.Tensor, ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) - q, k, v, gate = self._project_qkv_gate(qkv, positions) - assert gate is not None num_tokens = hidden_states.shape[0] - query = q.view(num_tokens, self.num_heads, self.head_dim) - key = k.view(num_tokens, self.num_kv_heads, self.head_dim) - value = v.view(num_tokens, self.num_kv_heads, self.head_dim) - attn_output = torch.empty_like(query) + if not self.use_fused_qsa_prepare: + q, k, v, gate = self._project_qkv_gate(qkv, positions) + assert gate is not None + query = q.view(num_tokens, self.num_heads, self.head_dim) + key = k.view(num_tokens, self.num_kv_heads, self.head_dim) + value = v.view(num_tokens, self.num_kv_heads, self.head_dim) + else: + # Norm/RoPE/gate and the K/V cache write happen inside _run_qsa. + query = key = value = gate = None + attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim) # Keep the index projection outside the eager break. projected_qk, _ = self.indexer.index_qk_proj(hidden_states) self._run_qsa( @@ -520,6 +541,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): value, attn_output, gate, + qkv, ) flat_output = attn_output.view(num_tokens, -1) output, _ = self.o_proj(flat_output)