diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index a25c9a78a8..fd21324a7e 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -216,7 +216,7 @@ steps: depends_on: - image-build-amd commands: - - pytest -v -s models/qwen4_exp/test_qsa_amd.py models/qwen4_exp/test_qsa_pre_indexer_amd.py + - pytest -v -s models/qwen4_exp/test_qsa_amd.py models/qwen4_exp/test_qsa_prepare_amd.py # PLE and QSA import conflicting HC registrations in a shared process. - pytest -v -s models/qwen4_exp/test_ple.py diff --git a/tests/models/qwen4_exp/test_qsa_amd.py b/tests/models/qwen4_exp/test_qsa_amd.py index 15c490223f..c8ebf01904 100644 --- a/tests/models/qwen4_exp/test_qsa_amd.py +++ b/tests/models/qwen4_exp/test_qsa_amd.py @@ -21,8 +21,8 @@ from vllm.models.qwen4_exp.amd.indexer_qsa import ( apply_qsa_rope, ) from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops -from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import ( - qsa_pre_indexer, +from vllm.models.qwen4_exp.amd.ops.qsa_prepare import ( + qsa_prepare, supports_fused_pre_indexer, ) from vllm.platforms import current_platform @@ -127,6 +127,18 @@ def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None: torch.testing.assert_close(output, norm.forward_native(tensor)) +def _make_attn(*, use_fused: bool) -> SimpleNamespace: + """The main-attention fields the fused QSA prepare reads from its owner.""" + return SimpleNamespace( + use_fused_qsa_prepare=use_fused, + kv_cache=torch.zeros(1, 2, 16, 8), + q_norm=GemmaRMSNorm(4, eps=1e-6), + k_norm=GemmaRMSNorm(4, eps=1e-6), + _k_scale_float=1.0, + _v_scale_float=1.0, + ) + + def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer: """A QSAIndexer whose unfused path runs for real, with no projection weights. @@ -184,29 +196,45 @@ def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer: @pytest.mark.usefixtures("default_vllm_config") -def test_qsa_fused_pre_indexer_replaces_the_unfused_chain( +def test_qsa_fused_prepare_replaces_the_unfused_chain( monkeypatch: pytest.MonkeyPatch, ) -> None: indexer = _make_indexer(use_fused=True) + attn = _make_attn(use_fused=True) calls = [] - monkeypatch.setattr( - indexer_qsa, - "qsa_pre_indexer", - lambda *args, **kwargs: calls.append((args, kwargs)), - ) + main_outputs = (torch.empty(0), torch.empty(0)) + + def fake_prepare(*args, **kwargs): + calls.append((args, kwargs)) + return main_outputs + + monkeypatch.setattr(indexer_qsa, "qsa_prepare", fake_prepare) monkeypatch.setattr( QSAIndexer, "_update_and_compress", lambda *args, **kwargs: pytest.fail("fused path ran the unfused chain"), ) rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32) + qkv = torch.zeros(4, 16) + slot_mapping = torch.arange(4) + gate_out = torch.empty(4, 1, 4) - actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows) + actual, actual_main = QSAIndexer.forward( + indexer, + torch.zeros(2, 8), + torch.tensor([7, 8]), + rows, + attn=attn, + qkv=qkv, + slot_mapping=slot_mapping, + gate_out=gate_out, + ) assert actual is rows + assert actual_main is main_outputs assert len(calls) == 1 args, kwargs = calls[0] - passed = inspect.signature(qsa_pre_indexer).bind(*args, **kwargs).arguments + passed = inspect.signature(qsa_prepare).bind(*args, **kwargs).arguments _, compressed_metadata = indexer.test_metadata # The kernel addresses the raw cache at its full width and writes the MRoPE # position tail itself, so it needs kv_cache, not the key_cache view. @@ -215,6 +243,17 @@ def test_qsa_fused_pre_indexer_replaces_the_unfused_chain( assert passed["k_work_metadata"] is compressed_metadata.k_work_metadata assert passed["compress_ratio"] == indexer.compress_ratio assert passed["rope_pos_offset"] == indexer.index_head_dim + # Main attention: only the real tokens, the vLLM cache viewed per token, + # and the gate landing in the caller's buffer. + assert passed["main_qkv"].shape[0] == 2 + assert passed["main_qkv"].data_ptr() == qkv.data_ptr() + assert passed["main_slot_mapping"].shape == (2,) + assert passed["main_kv_cache"].shape == (1, 16, 2, 8) + assert passed["main_kv_cache"].data_ptr() == attn.kv_cache.data_ptr() + assert passed["main_gate_out"].shape[0] == 2 + assert passed["main_gate_out"].data_ptr() == gate_out.data_ptr() + assert passed["main_q_norm_weight"] is attn.q_norm.weight + assert passed["main_k_norm_weight"] is attn.k_norm.weight @pytest.mark.usefixtures("default_vllm_config") @@ -224,7 +263,7 @@ def test_qsa_unsupported_config_keeps_the_unfused_chain( indexer = _make_indexer(use_fused=False) monkeypatch.setattr( indexer_qsa, - "qsa_pre_indexer", + "qsa_prepare", lambda *args, **kwargs: pytest.fail("unfused path ran the fused kernel"), ) updates = [] @@ -235,9 +274,16 @@ def test_qsa_unsupported_config_keeps_the_unfused_chain( ) rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32) - actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows) + actual, main_outputs = QSAIndexer.forward( + indexer, + torch.zeros(2, 8), + torch.tensor([7, 8]), + rows, + attn=_make_attn(use_fused=False), + ) assert actual is rows + assert main_outputs is None assert len(updates) == 1 diff --git a/tests/models/qwen4_exp/test_qsa_pre_indexer_amd.py b/tests/models/qwen4_exp/test_qsa_prepare_amd.py similarity index 80% rename from tests/models/qwen4_exp/test_qsa_pre_indexer_amd.py rename to tests/models/qwen4_exp/test_qsa_prepare_amd.py index 7698198923..7189297209 100644 --- a/tests/models/qwen4_exp/test_qsa_pre_indexer_amd.py +++ b/tests/models/qwen4_exp/test_qsa_prepare_amd.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Correctness tests for the AMD fused QSA pre-indexer.""" +"""Correctness tests for the AMD fused QSA prepare kernel.""" import pytest import torch @@ -11,9 +11,7 @@ from vllm.models.qwen4_exp.amd.ops.qsa import ( qsa_compress_groups_with_ratio, qsa_store_cache_rows, ) -from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import ( - qsa_pre_indexer, -) +from vllm.models.qwen4_exp.amd.ops.qsa_prepare import qsa_prepare from vllm.models.qwen4_exp.common.qsa_cache import ( canonical_qsa_rope_positions, circular_qsa_slot_mapping, @@ -52,6 +50,7 @@ ROPE_POS_OFFSET = D RTOL = 1.6e-2 ATOL = 1e-2 MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8]) +MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16 def _make_block_table(block_counts): @@ -78,6 +77,76 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all()) +def _make_main_inputs(num_tokens: int) -> dict: + """Random main-attention arguments with a paged BF16 cache in vLLM's layout.""" + num_blocks = num_tokens // MAIN_PAGE + 2 + cache = torch.zeros( + num_blocks, + MAIN_HK, + MAIN_PAGE, + 2 * MAIN_D, + dtype=torch.bfloat16, + device="cuda", + ).transpose(1, 2) + slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens] + slots[-1] = -1 + norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2 + return dict( + main_qkv=torch.randn( + num_tokens, + 2 * (MAIN_HQ + MAIN_HK) * MAIN_D, + dtype=torch.bfloat16, + device="cuda", + ), + main_q_norm_weight=norm_weights[0], + main_k_norm_weight=norm_weights[1], + main_eps=EPS, + main_kv_cache=cache, + main_slot_mapping=slots, + main_k_scale=1.0, + main_v_scale=1.0, + # The AMD op hands the gate back through a buffer it does not own. + main_gate_out=torch.empty( + num_tokens, MAIN_HQ, MAIN_D, dtype=torch.bfloat16, device="cuda" + ), + ) + + +def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None: + """Compare with the AMD layer's unfused split, QK-norm, RoPE and cache write.""" + from vllm._custom_ops import reshape_and_cache_flash + + num_tokens = main["main_qkv"].shape[0] + q_gate, k, v = main["main_qkv"].split( + [2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1 + ) + q, gate = torch.chunk(q_gate.view(num_tokens, MAIN_HQ, -1), 2, dim=-1) + q = _reference_gemma_rmsnorm( + q.reshape(-1, MAIN_D), main["main_q_norm_weight"], EPS + ).view(num_tokens, -1) + k = _reference_gemma_rmsnorm( + k.reshape(-1, MAIN_D), main["main_k_norm_weight"], EPS + ).view(num_tokens, -1) + q, k = rope(positions, q, k) + kv_cache = main["main_kv_cache"] + cache = torch.zeros_like(kv_cache) + key_cache, value_cache = cache.split(MAIN_D, dim=-1) + scale = torch.tensor(1.0, device="cuda") + reshape_and_cache_flash( + k.view(-1, MAIN_HK, MAIN_D), + v.view(-1, MAIN_HK, MAIN_D), + key_cache, + value_cache, + main["main_slot_mapping"], + "auto", + scale, + scale, + ) + torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL) + assert torch.equal(gate_out, gate) + torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL) + + @requires_qsa_kernels @pytest.mark.usefixtures("default_vllm_config") @pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn]) @@ -103,7 +172,7 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"), ], ) -def test_qsa_fused_pre_indexer_matches_unfused( +def test_qsa_fused_prepare_matches_unfused( indexer_dtype, mrope, is_2d_positions, @@ -269,7 +338,8 @@ def test_qsa_fused_pre_indexer_matches_unfused( k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2 fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device) - qsa_pre_indexer( + main = _make_main_inputs(num_tokens) + main_q_out, main_gate_out = qsa_prepare( projected_qk[:, : HQ * D], projected_qk[:, HQ * D :], positions, @@ -289,7 +359,10 @@ def test_qsa_fused_pre_indexer_matches_unfused( compress_ratio=CR, mrope_section=MROPE_SECTION if mrope else None, rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None, + **main, ) + assert main_gate_out is main["main_gate_out"] + _check_main_outputs(main, main_q_out, main_gate_out, rope, positions) unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D) unfused_query = _reference_gemma_rmsnorm( diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index fc371d94b4..7a16e3d6a0 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -774,6 +774,8 @@ class CompilationConfig: # Qwen4Exp's AMD backend still uses these splitting ops. "vllm::qwen4_exp_ple_short_conv", "vllm::qwen4_exp_qsa_with_output", + # The fused prepare calls this op instead of the one above. + "vllm::qwen4_exp_qsa_prepare_with_output", "vllm::linear_attention", "vllm::qwen_gdn_attention_core", "vllm::qwen_gdn_attention_core_fused_norm_packed", diff --git a/vllm/models/qwen4_exp/amd/indexer_qsa.py b/vllm/models/qwen4_exp/amd/indexer_qsa.py index 162914f39e..09cb39acf9 100644 --- a/vllm/models/qwen4_exp/amd/indexer_qsa.py +++ b/vllm/models/qwen4_exp/amd/indexer_qsa.py @@ -4,7 +4,7 @@ from __future__ import annotations -from typing import cast +from typing import TYPE_CHECKING, cast import torch from torch import nn @@ -23,7 +23,10 @@ from ..common.qsa_cache import ( QSAKeyStateCache, canonical_qsa_rope_positions, ) -from .ops.qsa_pre_indexer import qsa_pre_indexer, supports_fused_pre_indexer +from .ops.qsa_prepare import qsa_prepare, supports_fused_pre_indexer + +if TYPE_CHECKING: + from .qsa import Qwen4ExpQSAAttention def apply_qsa_rope( @@ -265,26 +268,31 @@ class QSAIndexer(nn.Module): position_rows, ) - def _fused_pre_indexer( + def _fused_prepare( self, projected_q: torch.Tensor, raw_keys: torch.Tensor, positions: torch.Tensor, raw_metadata: QSAForwardMetadata, compressed_metadata: QSAForwardMetadata, - ) -> torch.Tensor: - """Normalize and rotate Q, compress K, and write both caches in one launch. + attn: Qwen4ExpQSAAttention, + qkv: torch.Tensor, + slot_mapping: torch.Tensor, + gate_out: torch.Tensor | None, + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]: + """Prepare the indexer and ``attn``'s main attention in one launch. - Replaces ``project_qk``'s norm/RoPE half and the whole of - ``_update_and_compress``; the two must stay numerically interchangeable. + Normalizes and rotates the index Q, compresses K and writes both side + caches, which replaces ``project_qk``'s norm/RoPE half and the whole of + ``_update_and_compress``. The same launch applies ``attn``'s QK-norm and + RoPE, copies its gate and writes its K/V into ``attn.kv_cache``, + replacing ``_project_qkv_gate`` and ``do_kv_cache_update``. Both halves + must stay numerically interchangeable with what they replace. """ + num_tokens = raw_metadata.num_actual_tokens raw_key_cache = self.raw_key_cache - q = projected_q.new_empty( - raw_metadata.num_actual_tokens, - self.index_n_heads, - self.index_head_dim, - ) - qsa_pre_indexer( + q = projected_q.new_empty(num_tokens, self.index_n_heads, self.index_head_dim) + main_outputs = qsa_prepare( projected_q, raw_keys, positions, @@ -308,8 +316,17 @@ class QSAIndexer(nn.Module): if raw_key_cache.cache_rope_positions else None ), + main_qkv=qkv[:num_tokens], + main_q_norm_weight=attn.q_norm.weight, + main_k_norm_weight=attn.k_norm.weight, + main_eps=attn.q_norm.variance_epsilon, + main_kv_cache=attn.kv_cache.transpose(1, 2), + main_slot_mapping=slot_mapping[:num_tokens], + main_k_scale=attn._k_scale_float, + main_v_scale=attn._v_scale_float, + main_gate_out=None if gate_out is None else gate_out[:num_tokens], ) - return q + return q, main_outputs def _select( self, @@ -336,13 +353,25 @@ class QSAIndexer(nn.Module): hidden_states: torch.Tensor, positions: torch.Tensor, out: torch.Tensor | None = None, - ) -> torch.Tensor: - """Return fixed-width request-relative token indices padded with ``-1``.""" + *, + attn: Qwen4ExpQSAAttention, + qkv: torch.Tensor | None = None, + slot_mapping: torch.Tensor | None = None, + gate_out: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]: + """Return fixed-width request-relative token indices padded with ``-1``. + + With ``attn.use_fused_qsa_prepare``, the same launch also writes + ``attn``'s K/V into ``attn.kv_cache`` at ``slot_mapping`` and prepares + its Q and gate from ``qkv``, returned as the second element (None + otherwise). The gate is written into ``gate_out`` when given. ``qkv``, + ``slot_mapping`` and ``gate_out`` are only read in that mode. + """ metadata = self._metadata() if metadata is None: # Preserve step-0 indices when later MTP steps reuse the buffer. if self.skip_topk and out is not None: - return out + return out, None result = torch.full( (hidden_states.shape[0], self.output_width), -1, @@ -351,20 +380,27 @@ class QSAIndexer(nn.Module): ) if out is not None: out.copy_(result) - return out - return result + return out, None + return result, None raw_metadata, compressed_metadata = metadata num_tokens = raw_metadata.num_actual_tokens hidden_states = hidden_states[:num_tokens] positions = positions[..., :num_tokens] - if self.use_fused_pre_indexer: + main_outputs: tuple[torch.Tensor, torch.Tensor] | None = None + if attn.use_fused_qsa_prepare: + if qkv is None or slot_mapping is None: + raise ValueError("fused QSA prepare requires qkv and slot_mapping") projected_q, raw_keys = self.project(hidden_states) - q = self._fused_pre_indexer( + q, main_outputs = self._fused_prepare( projected_q, raw_keys, positions, raw_metadata, compressed_metadata, + attn, + qkv, + slot_mapping, + gate_out, ) else: q, token_k = self.project_qk(hidden_states, positions) @@ -377,8 +413,8 @@ class QSAIndexer(nn.Module): if self.skip_topk: if out is None: raise RuntimeError("QSA top-k reuse requires an output buffer") - return out - return self._select(q, compressed_metadata, out) + return out, main_outputs + return self._select(q, compressed_metadata, out), main_outputs __all__ = ["QSAIndexer", "apply_qsa_rope"] diff --git a/vllm/models/qwen4_exp/amd/ops/qsa_pre_indexer.py b/vllm/models/qwen4_exp/amd/ops/qsa_prepare.py similarity index 75% rename from vllm/models/qwen4_exp/amd/ops/qsa_pre_indexer.py rename to vllm/models/qwen4_exp/amd/ops/qsa_prepare.py index 6a4873882f..bc2fee3b3a 100644 --- a/vllm/models/qwen4_exp/amd/ops/qsa_pre_indexer.py +++ b/vllm/models/qwen4_exp/amd/ops/qsa_prepare.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Fused QSA pre-indexer kernel for the AMD Qwen4Exp path. +"""Fused QSA prepare kernel for the AMD Qwen4Exp path. Started as a copy of the NVIDIA kernel (``nvidia/ops/qsa_prepare.py``) and is kept separate so either vendor can change its own copy without re-validating @@ -119,6 +119,27 @@ def _norm_rope( return tl.reshape(result, (TILE_T, TILE_H, D)) +@triton.jit +def _to_dst_dtype(x, dst, scale): + """Round to BF16 like the unfused path, then scale for an FP8 destination.""" + out_ty = dst.dtype.element_ty + x = x.to(tl.bfloat16) + if out_ty == tl.float8e4nv: + x = x.to(tl.float32) / scale + return x.to(out_ty) + + +@triton.jit +def _store_rotated(dst, y, o1, o2, scale): + """Store a normalized head whose first ``2 * len(o1)`` dims are rotated.""" + HALF: tl.constexpr = o1.shape[0] + dims = tl.arange(0, y.shape[0]) + rot = tl.arange(0, HALF) + tl.store(dst + dims, _to_dst_dtype(y, dst, scale), mask=dims >= 2 * HALF) + tl.store(dst + rot, _to_dst_dtype(o1, dst, scale)) + tl.store(dst + HALF + rot, _to_dst_dtype(o2, dst, scale)) + + @triton.jit( do_not_specialize=[ "num_tokens", @@ -127,7 +148,7 @@ def _norm_rope( "num_k_work", ] ) -def _qsa_pre_indexer_kernel( +def _qsa_prepare_kernel( q_ptr, q_stride_token, k_ptr, @@ -171,8 +192,86 @@ def _qsa_pre_indexer_kernel( CACHE_HAS_ROPE_POS: tl.constexpr, MROPE_H: tl.constexpr, MROPE_W: tl.constexpr, + main_qkv_ptr, + main_q_norm_weight_ptr, + main_k_norm_weight_ptr, + main_eps, + main_q_out_ptr, + main_gate_out_ptr, + main_cache_ptr, + main_cache_stride_block, + main_cache_stride_token, + main_cache_stride_head, + main_slots_ptr, + main_k_scale, + main_v_scale, + MAIN_HQ: tl.constexpr, + MAIN_HK: tl.constexpr, + MAIN_D: tl.constexpr, + MAIN_PAGE_SIZE: tl.constexpr, ): pid = tl.program_id(0) + num_index_work = num_k_work + tl.cdiv(num_tokens, TILE_T_Q) * tl.cdiv(HQ, TILE_H_Q) + if pid >= num_index_work: + # Main attention: one program per (token, Q or KV head) after the + # indexer work. RoPE covers the first D // 2 dims, the width of the + # shared cos/sin table. + main_pid = pid - num_index_work + token = (main_pid // (MAIN_HQ + MAIN_HK)).to(tl.int64) + head = main_pid % (MAIN_HQ + MAIN_HK) + HALF: tl.constexpr = D // 4 + dims = tl.arange(0, MAIN_D) + rot = tl.arange(0, HALF) + row = main_qkv_ptr + token * (2 * (MAIN_HQ + MAIN_HK) * MAIN_D) + is_k = head >= MAIN_HQ + kv_head = head - MAIN_HQ + if is_k: + src = row + (2 * MAIN_HQ + kv_head) * MAIN_D + weight_ptr = main_k_norm_weight_ptr + else: + src = row + 2 * head * MAIN_D + weight_ptr = main_q_norm_weight_ptr + x = tl.load(src + dims).to(tl.float32) + inv_rms = tl.rsqrt(tl.sum(x * x, axis=0) / MAIN_D + main_eps) + w = tl.load(weight_ptr + dims).to(tl.float32) + 1.0 + y = x * inv_rms * w + x1 = tl.load(src + rot).to(tl.float32) + x2 = tl.load(src + HALF + rot).to(tl.float32) + w1 = tl.load(weight_ptr + rot).to(tl.float32) + 1.0 + w2 = tl.load(weight_ptr + HALF + rot).to(tl.float32) + 1.0 + x1 = (x1 * inv_rms * w1).to(tl.bfloat16).to(tl.float32) + x2 = (x2 * inv_rms * w2).to(tl.bfloat16).to(tl.float32) + pos = tl.load(pos_ptr + token * pos_stride_token).to(tl.int64) + if IS_2D_POSITIONS: + pos_h = tl.load(pos_ptr + pos_stride_axis + token * pos_stride_token) + pos_w = tl.load(pos_ptr + 2 * pos_stride_axis + token * pos_stride_token) + is_h = (rot % 3 == 1) & (rot < 3 * MROPE_H) + is_w = (rot % 3 == 2) & (rot < 3 * MROPE_W) + pos = tl.where( + is_h, pos_h.to(tl.int64), tl.where(is_w, pos_w.to(tl.int64), pos) + ) + cos = tl.load(cos_sin_ptr + pos * (D // 2) + rot).to(tl.float32) + sin = tl.load(cos_sin_ptr + pos * (D // 2) + HALF + rot).to(tl.float32) + o1 = x1 * cos - x2 * sin + o2 = x2 * cos + x1 * sin + if is_k: + slot = tl.load(main_slots_ptr + token).to(tl.int64) + if slot >= 0: + dst = ( + main_cache_ptr + + (slot // MAIN_PAGE_SIZE) * main_cache_stride_block + + (slot % MAIN_PAGE_SIZE) * main_cache_stride_token + + kv_head * main_cache_stride_head + ) + _store_rotated(dst, y, o1, o2, main_k_scale) + v = tl.load(src + MAIN_HK * MAIN_D + dims) + tl.store(dst + MAIN_D + dims, _to_dst_dtype(v, dst, main_v_scale)) + else: + out = (token * MAIN_HQ + head) * MAIN_D + _store_rotated(main_q_out_ptr + out, y, o1, o2, None) + gate = tl.load(src + MAIN_D + dims) + tl.store(main_gate_out_ptr + out + dims, gate) + return # K work occupies the first programs; the remaining programs tile Q. This # keeps both paths in one launch while leaving their register shapes # independent. @@ -181,7 +280,7 @@ def _qsa_pre_indexer_kernel( num_head_tiles: tl.constexpr = tl.cdiv(HQ, TILE_H_Q) token_tile = q_pid // num_head_tiles head_tile = q_pid % num_head_tiles - tokens = token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q) + tokens = (token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q)).to(tl.int64) heads = head_tile * TILE_H_Q + tl.arange(0, TILE_H_Q) valid_tokens = tokens < num_tokens valid_heads = heads < HQ @@ -283,7 +382,8 @@ def _qsa_pre_indexer_kernel( & (source_tokens < num_tokens) ) current_base = ( - k_ptr + tl.maximum(source_tokens, 0)[:, None] * k_stride_token + k_ptr + + tl.maximum(source_tokens, 0).to(tl.int64)[:, None] * k_stride_token ) cached_base = ( state_cache_ptr @@ -397,7 +497,7 @@ def _qsa_pre_indexer_kernel( # One CTA per request commits only the suffix retained by the ring. num_state_rows = tl.minimum(query_len, STATE_SIZE) for state_offset in tl.range(0, num_state_rows): - token = query_end - num_state_rows + state_offset + token = (query_end - num_state_rows + state_offset).to(tl.int64) valid_token = ( (token >= query_start) & (token < query_end) & (token < num_tokens) ) @@ -447,7 +547,7 @@ def _qsa_pre_indexer_kernel( tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot) -def qsa_pre_indexer( +def qsa_prepare( q: torch.Tensor, k: torch.Tensor, positions: torch.Tensor, @@ -468,11 +568,34 @@ def qsa_pre_indexer( compress_ratio: int, mrope_section: tuple[int, int, int] | None, rope_pos_offset: int | None, -) -> None: - """Normalize Q, compress K, then update the circular raw state.""" + main_qkv: torch.Tensor, + main_q_norm_weight: torch.Tensor, + main_k_norm_weight: torch.Tensor, + main_eps: float, + main_kv_cache: torch.Tensor, + main_slot_mapping: torch.Tensor, + main_k_scale: float, + main_v_scale: float, + main_gate_out: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Normalize Q, compress K, then update the circular raw state. + + Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write) + and returns its Q and gate. ``main_gate_out`` lets the gate land in a + caller-owned buffer, e.g. one that outlives an opaque custom op. + """ num_tokens = q.shape[0] + main_head_dim = main_kv_cache.shape[-1] // 2 + num_main_kv_heads = main_kv_cache.shape[2] + num_main_q_heads = main_qkv.shape[1] // (2 * main_head_dim) - num_main_kv_heads + main_q_out = main_qkv.new_empty(num_tokens, num_main_q_heads, main_head_dim) + if main_gate_out is None: + main_gate_out = torch.empty_like(main_q_out) + else: + assert main_gate_out.shape == main_q_out.shape + assert main_gate_out.is_contiguous() if num_tokens == 0: - return + return main_q_out, main_gate_out num_q_heads, head_dim = q_out.shape[1:] assert cos_sin_cache.shape[-1] * 2 == head_dim assert q.shape == (num_tokens, num_q_heads * head_dim) @@ -497,6 +620,9 @@ def qsa_pre_indexer( pos_stride_axis, pos_stride_token = 0, positions.stride(0) section = mrope_section if mrope_section is not None else (0, 0, 0) assert len(section) == 3 + qkv_width = 2 * (num_main_q_heads + num_main_kv_heads) * main_head_dim + assert main_qkv.shape == (num_tokens, qkv_width) and main_qkv.is_contiguous() + assert main_slot_mapping.shape == (num_tokens,) # Swept on MI355X (64-lane wavefronts) over TILE_T_Q 1-16, TILE_H_Q 1-4 and # num_warps 1/2/4, from 4 to 16384 tokens: no config beat these by more @@ -507,7 +633,8 @@ def qsa_pre_indexer( TILE_T_Q, TILE_H_Q = 2, 4 num_k_work = k_work_metadata.shape[0] num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q) - _qsa_pre_indexer_kernel[(num_k_work + num_q_work,)]( + num_main_work = num_tokens * (num_main_q_heads + num_main_kv_heads) + _qsa_prepare_kernel[(num_k_work + num_q_work + num_main_work,)]( q, q.stride(0), k, @@ -551,8 +678,26 @@ def qsa_pre_indexer( CACHE_HAS_ROPE_POS=cache_has_rope_pos, MROPE_H=section[1], MROPE_W=section[2], + main_qkv_ptr=main_qkv, + main_q_norm_weight_ptr=main_q_norm_weight, + main_k_norm_weight_ptr=main_k_norm_weight, + main_eps=main_eps, + main_q_out_ptr=main_q_out, + main_gate_out_ptr=main_gate_out, + main_cache_ptr=main_kv_cache, + main_cache_stride_block=main_kv_cache.stride(0), + main_cache_stride_token=main_kv_cache.stride(1), + main_cache_stride_head=main_kv_cache.stride(2), + main_slots_ptr=main_slot_mapping, + main_k_scale=main_k_scale, + main_v_scale=main_v_scale, + MAIN_HQ=num_main_q_heads, + MAIN_HK=num_main_kv_heads, + MAIN_D=main_head_dim, + MAIN_PAGE_SIZE=main_kv_cache.shape[1], num_warps=1, ) + return main_q_out, main_gate_out -__all__ = ["qsa_pre_indexer", "supports_fused_pre_indexer"] +__all__ = ["qsa_prepare", "supports_fused_pre_indexer"] diff --git a/vllm/models/qwen4_exp/amd/qsa.py b/vllm/models/qwen4_exp/amd/qsa.py index 3f03b293b4..ff01490bb1 100644 --- a/vllm/models/qwen4_exp/amd/qsa.py +++ b/vllm/models/qwen4_exp/amd/qsa.py @@ -116,8 +116,8 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl): self, layer: torch.nn.Module, query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, + key: torch.Tensor | None, + value: torch.Tensor | None, kv_cache: torch.Tensor, attn_metadata: FlashAttentionMetadata, output: torch.Tensor, @@ -294,6 +294,11 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): quant_config=quant_config, prefix=f"{prefix}.indexer", ) + self.use_fused_qsa_prepare = ( + self.attn_output_gate + and self.indexer.use_fused_pre_indexer + and self.head_dim & (self.head_dim - 1) == 0 + ) max_tokens = vllm_config.scheduler_config.max_num_batched_tokens self.register_buffer( "topk_indices_buffer", @@ -327,16 +332,22 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): self, hidden_states: torch.Tensor, positions: torch.Tensor, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, + query: torch.Tensor | None, + key: torch.Tensor | None, + value: torch.Tensor | None, output: torch.Tensor, + qkv: torch.Tensor | None = None, + gate: torch.Tensor | None = None, ) -> None: + # query/key/value are None when the fused prepare runs inside the + # indexer launch, which then writes the gate into ``gate``. metadata = get_forward_context().attn_metadata if isinstance(metadata, list): metadata = metadata[0] if not isinstance(metadata, dict): output.zero_() + if gate is not None: + gate.zero_() return main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name]) if self.kv_cache.numel() == 0: @@ -349,10 +360,14 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): ) if side_metadata.num_actual_tokens != num_tokens: raise RuntimeError("QSA main and side metadata token counts disagree") - selected = self.indexer( + selected, main_outputs = self.indexer( hidden_states, positions, self.topk_indices_buffer[:num_tokens], + attn=self, + qkv=qkv, + slot_mapping=main_metadata.slot_mapping, + gate_out=gate, ) if selected.shape != ( num_tokens, @@ -360,13 +375,22 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): ): raise RuntimeError("QSA indexer returned an invalid selection shape") impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl) - impl.do_kv_cache_update( - self, - key, - value, - self.kv_cache, - main_metadata.slot_mapping, - ) + if main_outputs is None: + assert key is not None and value is not None + impl.do_kv_cache_update( + self, + key, + value, + self.kv_cache, + main_metadata.slot_mapping, + ) + else: + query, _ = main_outputs + # The gate is applied to every row outside this op; padded rows + # must stay finite so they keep the zero output forward_qsa gives. + assert gate is not None + gate[num_tokens:].zero_() + assert query is not None impl.forward_qsa( self, query, @@ -384,13 +408,40 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): hidden_states: torch.Tensor, ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) - q, k, v, gate = self._project_qkv_gate(qkv, positions) num_tokens = hidden_states.shape[0] + encoded_layer_name = _encode_layer_name(self.layer_name) + if self.use_fused_qsa_prepare: + # Norm/RoPE/gate and the K/V cache write happen inside the op. + attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim) + gate = torch.empty_like(attn_output) + if current_platform.opaque_attention_op(): + torch.ops.vllm.qwen4_exp_qsa_prepare_with_output( + hidden_states, + positions, + qkv, + attn_output, + gate, + encoded_layer_name, + ) + else: + qwen4_exp_qsa_prepare_with_output( + hidden_states, + positions, + qkv, + attn_output, + gate, + encoded_layer_name, + ) + flat_output = attn_output.view(num_tokens, -1) + flat_output = flat_output * torch.sigmoid(gate.view(num_tokens, -1)) + output, _ = self.o_proj(flat_output) + return output + + q, k, v, gate = self._project_qkv_gate(qkv, positions) query = q.view(num_tokens, self.num_heads, self.head_dim) key = k.view(num_tokens, self.num_kv_heads, self.head_dim) value = v.view(num_tokens, self.num_kv_heads, self.head_dim) attn_output = torch.empty_like(query) - encoded_layer_name = _encode_layer_name(self.layer_name) if current_platform.opaque_attention_op(): torch.ops.vllm.qwen4_exp_qsa_with_output( hidden_states, @@ -449,10 +500,43 @@ direct_register_custom_op( ) +def qwen4_exp_qsa_prepare_with_output( + hidden_states: torch.Tensor, + positions: torch.Tensor, + qkv: torch.Tensor, + output: torch.Tensor, + gate: torch.Tensor, + layer_name: LayerNameType, +) -> None: + """Run the QSA transaction with the main QK-norm/RoPE/gate fused in.""" + layer_name = _resolve_layer_name(layer_name) + layer = get_forward_context().no_compile_layers[layer_name] + if not isinstance(layer, Qwen4ExpQSAAttention): + raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner") + layer._run_qsa( + hidden_states, + positions, + None, + None, + None, + output, + qkv=qkv, + gate=gate, + ) + + +direct_register_custom_op( + op_name="qwen4_exp_qsa_prepare_with_output", + op_func=qwen4_exp_qsa_prepare_with_output, + mutates_args=["output", "gate"], +) + + __all__ = [ "QSAIndexer", "Qwen4ExpQSAAttention", "Qwen4ExpQSAFlashAttentionBackend", "Qwen4ExpQSAFlashAttentionImpl", + "qwen4_exp_qsa_prepare_with_output", "qwen4_exp_qsa_with_output", ]