[ROCm][Perf] Fuse main QK-norm/RoPE/gate and KV-cache write into the AMD QSA prepare launch (#59732)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
9adba10f3f
commit
81c993658e
@@ -216,7 +216,7 @@ steps:
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
commands:
|
||||
- pytest -v -s models/qwen4_exp/test_qsa_amd.py models/qwen4_exp/test_qsa_pre_indexer_amd.py
|
||||
- pytest -v -s models/qwen4_exp/test_qsa_amd.py models/qwen4_exp/test_qsa_prepare_amd.py
|
||||
# PLE and QSA import conflicting HC registrations in a shared process.
|
||||
- pytest -v -s models/qwen4_exp/test_ple.py
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@ from vllm.models.qwen4_exp.amd.indexer_qsa import (
|
||||
apply_qsa_rope,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import (
|
||||
qsa_pre_indexer,
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa_prepare import (
|
||||
qsa_prepare,
|
||||
supports_fused_pre_indexer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -127,6 +127,18 @@ def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
|
||||
torch.testing.assert_close(output, norm.forward_native(tensor))
|
||||
|
||||
|
||||
def _make_attn(*, use_fused: bool) -> SimpleNamespace:
|
||||
"""The main-attention fields the fused QSA prepare reads from its owner."""
|
||||
return SimpleNamespace(
|
||||
use_fused_qsa_prepare=use_fused,
|
||||
kv_cache=torch.zeros(1, 2, 16, 8),
|
||||
q_norm=GemmaRMSNorm(4, eps=1e-6),
|
||||
k_norm=GemmaRMSNorm(4, eps=1e-6),
|
||||
_k_scale_float=1.0,
|
||||
_v_scale_float=1.0,
|
||||
)
|
||||
|
||||
|
||||
def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer:
|
||||
"""A QSAIndexer whose unfused path runs for real, with no projection weights.
|
||||
|
||||
@@ -184,29 +196,45 @@ def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer:
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_qsa_fused_pre_indexer_replaces_the_unfused_chain(
|
||||
def test_qsa_fused_prepare_replaces_the_unfused_chain(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
indexer = _make_indexer(use_fused=True)
|
||||
attn = _make_attn(use_fused=True)
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)),
|
||||
)
|
||||
main_outputs = (torch.empty(0), torch.empty(0))
|
||||
|
||||
def fake_prepare(*args, **kwargs):
|
||||
calls.append((args, kwargs))
|
||||
return main_outputs
|
||||
|
||||
monkeypatch.setattr(indexer_qsa, "qsa_prepare", fake_prepare)
|
||||
monkeypatch.setattr(
|
||||
QSAIndexer,
|
||||
"_update_and_compress",
|
||||
lambda *args, **kwargs: pytest.fail("fused path ran the unfused chain"),
|
||||
)
|
||||
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
|
||||
qkv = torch.zeros(4, 16)
|
||||
slot_mapping = torch.arange(4)
|
||||
gate_out = torch.empty(4, 1, 4)
|
||||
|
||||
actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows)
|
||||
actual, actual_main = QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 8),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
attn=attn,
|
||||
qkv=qkv,
|
||||
slot_mapping=slot_mapping,
|
||||
gate_out=gate_out,
|
||||
)
|
||||
|
||||
assert actual is rows
|
||||
assert actual_main is main_outputs
|
||||
assert len(calls) == 1
|
||||
args, kwargs = calls[0]
|
||||
passed = inspect.signature(qsa_pre_indexer).bind(*args, **kwargs).arguments
|
||||
passed = inspect.signature(qsa_prepare).bind(*args, **kwargs).arguments
|
||||
_, compressed_metadata = indexer.test_metadata
|
||||
# The kernel addresses the raw cache at its full width and writes the MRoPE
|
||||
# position tail itself, so it needs kv_cache, not the key_cache view.
|
||||
@@ -215,6 +243,17 @@ def test_qsa_fused_pre_indexer_replaces_the_unfused_chain(
|
||||
assert passed["k_work_metadata"] is compressed_metadata.k_work_metadata
|
||||
assert passed["compress_ratio"] == indexer.compress_ratio
|
||||
assert passed["rope_pos_offset"] == indexer.index_head_dim
|
||||
# Main attention: only the real tokens, the vLLM cache viewed per token,
|
||||
# and the gate landing in the caller's buffer.
|
||||
assert passed["main_qkv"].shape[0] == 2
|
||||
assert passed["main_qkv"].data_ptr() == qkv.data_ptr()
|
||||
assert passed["main_slot_mapping"].shape == (2,)
|
||||
assert passed["main_kv_cache"].shape == (1, 16, 2, 8)
|
||||
assert passed["main_kv_cache"].data_ptr() == attn.kv_cache.data_ptr()
|
||||
assert passed["main_gate_out"].shape[0] == 2
|
||||
assert passed["main_gate_out"].data_ptr() == gate_out.data_ptr()
|
||||
assert passed["main_q_norm_weight"] is attn.q_norm.weight
|
||||
assert passed["main_k_norm_weight"] is attn.k_norm.weight
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@@ -224,7 +263,7 @@ def test_qsa_unsupported_config_keeps_the_unfused_chain(
|
||||
indexer = _make_indexer(use_fused=False)
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
"qsa_prepare",
|
||||
lambda *args, **kwargs: pytest.fail("unfused path ran the fused kernel"),
|
||||
)
|
||||
updates = []
|
||||
@@ -235,9 +274,16 @@ def test_qsa_unsupported_config_keeps_the_unfused_chain(
|
||||
)
|
||||
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
|
||||
|
||||
actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows)
|
||||
actual, main_outputs = QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 8),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
attn=_make_attn(use_fused=False),
|
||||
)
|
||||
|
||||
assert actual is rows
|
||||
assert main_outputs is None
|
||||
assert len(updates) == 1
|
||||
|
||||
|
||||
|
||||
+79
-6
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the AMD fused QSA pre-indexer."""
|
||||
"""Correctness tests for the AMD fused QSA prepare kernel."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -11,9 +11,7 @@ from vllm.models.qwen4_exp.amd.ops.qsa import (
|
||||
qsa_compress_groups_with_ratio,
|
||||
qsa_store_cache_rows,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import (
|
||||
qsa_pre_indexer,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa_prepare import qsa_prepare
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import (
|
||||
canonical_qsa_rope_positions,
|
||||
circular_qsa_slot_mapping,
|
||||
@@ -52,6 +50,7 @@ ROPE_POS_OFFSET = D
|
||||
RTOL = 1.6e-2
|
||||
ATOL = 1e-2
|
||||
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
|
||||
MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16
|
||||
|
||||
|
||||
def _make_block_table(block_counts):
|
||||
@@ -78,6 +77,76 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
|
||||
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
|
||||
|
||||
|
||||
def _make_main_inputs(num_tokens: int) -> dict:
|
||||
"""Random main-attention arguments with a paged BF16 cache in vLLM's layout."""
|
||||
num_blocks = num_tokens // MAIN_PAGE + 2
|
||||
cache = torch.zeros(
|
||||
num_blocks,
|
||||
MAIN_HK,
|
||||
MAIN_PAGE,
|
||||
2 * MAIN_D,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
).transpose(1, 2)
|
||||
slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens]
|
||||
slots[-1] = -1
|
||||
norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2
|
||||
return dict(
|
||||
main_qkv=torch.randn(
|
||||
num_tokens,
|
||||
2 * (MAIN_HQ + MAIN_HK) * MAIN_D,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
),
|
||||
main_q_norm_weight=norm_weights[0],
|
||||
main_k_norm_weight=norm_weights[1],
|
||||
main_eps=EPS,
|
||||
main_kv_cache=cache,
|
||||
main_slot_mapping=slots,
|
||||
main_k_scale=1.0,
|
||||
main_v_scale=1.0,
|
||||
# The AMD op hands the gate back through a buffer it does not own.
|
||||
main_gate_out=torch.empty(
|
||||
num_tokens, MAIN_HQ, MAIN_D, dtype=torch.bfloat16, device="cuda"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None:
|
||||
"""Compare with the AMD layer's unfused split, QK-norm, RoPE and cache write."""
|
||||
from vllm._custom_ops import reshape_and_cache_flash
|
||||
|
||||
num_tokens = main["main_qkv"].shape[0]
|
||||
q_gate, k, v = main["main_qkv"].split(
|
||||
[2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1
|
||||
)
|
||||
q, gate = torch.chunk(q_gate.view(num_tokens, MAIN_HQ, -1), 2, dim=-1)
|
||||
q = _reference_gemma_rmsnorm(
|
||||
q.reshape(-1, MAIN_D), main["main_q_norm_weight"], EPS
|
||||
).view(num_tokens, -1)
|
||||
k = _reference_gemma_rmsnorm(
|
||||
k.reshape(-1, MAIN_D), main["main_k_norm_weight"], EPS
|
||||
).view(num_tokens, -1)
|
||||
q, k = rope(positions, q, k)
|
||||
kv_cache = main["main_kv_cache"]
|
||||
cache = torch.zeros_like(kv_cache)
|
||||
key_cache, value_cache = cache.split(MAIN_D, dim=-1)
|
||||
scale = torch.tensor(1.0, device="cuda")
|
||||
reshape_and_cache_flash(
|
||||
k.view(-1, MAIN_HK, MAIN_D),
|
||||
v.view(-1, MAIN_HK, MAIN_D),
|
||||
key_cache,
|
||||
value_cache,
|
||||
main["main_slot_mapping"],
|
||||
"auto",
|
||||
scale,
|
||||
scale,
|
||||
)
|
||||
torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL)
|
||||
assert torch.equal(gate_out, gate)
|
||||
torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@@ -103,7 +172,7 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
|
||||
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
|
||||
],
|
||||
)
|
||||
def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
def test_qsa_fused_prepare_matches_unfused(
|
||||
indexer_dtype,
|
||||
mrope,
|
||||
is_2d_positions,
|
||||
@@ -269,7 +338,8 @@ def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
|
||||
qsa_pre_indexer(
|
||||
main = _make_main_inputs(num_tokens)
|
||||
main_q_out, main_gate_out = qsa_prepare(
|
||||
projected_qk[:, : HQ * D],
|
||||
projected_qk[:, HQ * D :],
|
||||
positions,
|
||||
@@ -289,7 +359,10 @@ def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
compress_ratio=CR,
|
||||
mrope_section=MROPE_SECTION if mrope else None,
|
||||
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
|
||||
**main,
|
||||
)
|
||||
assert main_gate_out is main["main_gate_out"]
|
||||
_check_main_outputs(main, main_q_out, main_gate_out, rope, positions)
|
||||
|
||||
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
|
||||
unfused_query = _reference_gemma_rmsnorm(
|
||||
@@ -774,6 +774,8 @@ class CompilationConfig:
|
||||
# Qwen4Exp's AMD backend still uses these splitting ops.
|
||||
"vllm::qwen4_exp_ple_short_conv",
|
||||
"vllm::qwen4_exp_qsa_with_output",
|
||||
# The fused prepare calls this op instead of the one above.
|
||||
"vllm::qwen4_exp_qsa_prepare_with_output",
|
||||
"vllm::linear_attention",
|
||||
"vllm::qwen_gdn_attention_core",
|
||||
"vllm::qwen_gdn_attention_core_fused_norm_packed",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -23,7 +23,10 @@ from ..common.qsa_cache import (
|
||||
QSAKeyStateCache,
|
||||
canonical_qsa_rope_positions,
|
||||
)
|
||||
from .ops.qsa_pre_indexer import qsa_pre_indexer, supports_fused_pre_indexer
|
||||
from .ops.qsa_prepare import qsa_prepare, supports_fused_pre_indexer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .qsa import Qwen4ExpQSAAttention
|
||||
|
||||
|
||||
def apply_qsa_rope(
|
||||
@@ -265,26 +268,31 @@ class QSAIndexer(nn.Module):
|
||||
position_rows,
|
||||
)
|
||||
|
||||
def _fused_pre_indexer(
|
||||
def _fused_prepare(
|
||||
self,
|
||||
projected_q: torch.Tensor,
|
||||
raw_keys: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
raw_metadata: QSAForwardMetadata,
|
||||
compressed_metadata: QSAForwardMetadata,
|
||||
) -> torch.Tensor:
|
||||
"""Normalize and rotate Q, compress K, and write both caches in one launch.
|
||||
attn: Qwen4ExpQSAAttention,
|
||||
qkv: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
gate_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Prepare the indexer and ``attn``'s main attention in one launch.
|
||||
|
||||
Replaces ``project_qk``'s norm/RoPE half and the whole of
|
||||
``_update_and_compress``; the two must stay numerically interchangeable.
|
||||
Normalizes and rotates the index Q, compresses K and writes both side
|
||||
caches, which replaces ``project_qk``'s norm/RoPE half and the whole of
|
||||
``_update_and_compress``. The same launch applies ``attn``'s QK-norm and
|
||||
RoPE, copies its gate and writes its K/V into ``attn.kv_cache``,
|
||||
replacing ``_project_qkv_gate`` and ``do_kv_cache_update``. Both halves
|
||||
must stay numerically interchangeable with what they replace.
|
||||
"""
|
||||
num_tokens = raw_metadata.num_actual_tokens
|
||||
raw_key_cache = self.raw_key_cache
|
||||
q = projected_q.new_empty(
|
||||
raw_metadata.num_actual_tokens,
|
||||
self.index_n_heads,
|
||||
self.index_head_dim,
|
||||
)
|
||||
qsa_pre_indexer(
|
||||
q = projected_q.new_empty(num_tokens, self.index_n_heads, self.index_head_dim)
|
||||
main_outputs = qsa_prepare(
|
||||
projected_q,
|
||||
raw_keys,
|
||||
positions,
|
||||
@@ -308,8 +316,17 @@ class QSAIndexer(nn.Module):
|
||||
if raw_key_cache.cache_rope_positions
|
||||
else None
|
||||
),
|
||||
main_qkv=qkv[:num_tokens],
|
||||
main_q_norm_weight=attn.q_norm.weight,
|
||||
main_k_norm_weight=attn.k_norm.weight,
|
||||
main_eps=attn.q_norm.variance_epsilon,
|
||||
main_kv_cache=attn.kv_cache.transpose(1, 2),
|
||||
main_slot_mapping=slot_mapping[:num_tokens],
|
||||
main_k_scale=attn._k_scale_float,
|
||||
main_v_scale=attn._v_scale_float,
|
||||
main_gate_out=None if gate_out is None else gate_out[:num_tokens],
|
||||
)
|
||||
return q
|
||||
return q, main_outputs
|
||||
|
||||
def _select(
|
||||
self,
|
||||
@@ -336,13 +353,25 @@ class QSAIndexer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Return fixed-width request-relative token indices padded with ``-1``."""
|
||||
*,
|
||||
attn: Qwen4ExpQSAAttention,
|
||||
qkv: torch.Tensor | None = None,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
gate_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]:
|
||||
"""Return fixed-width request-relative token indices padded with ``-1``.
|
||||
|
||||
With ``attn.use_fused_qsa_prepare``, the same launch also writes
|
||||
``attn``'s K/V into ``attn.kv_cache`` at ``slot_mapping`` and prepares
|
||||
its Q and gate from ``qkv``, returned as the second element (None
|
||||
otherwise). The gate is written into ``gate_out`` when given. ``qkv``,
|
||||
``slot_mapping`` and ``gate_out`` are only read in that mode.
|
||||
"""
|
||||
metadata = self._metadata()
|
||||
if metadata is None:
|
||||
# Preserve step-0 indices when later MTP steps reuse the buffer.
|
||||
if self.skip_topk and out is not None:
|
||||
return out
|
||||
return out, None
|
||||
result = torch.full(
|
||||
(hidden_states.shape[0], self.output_width),
|
||||
-1,
|
||||
@@ -351,20 +380,27 @@ class QSAIndexer(nn.Module):
|
||||
)
|
||||
if out is not None:
|
||||
out.copy_(result)
|
||||
return out
|
||||
return result
|
||||
return out, None
|
||||
return result, None
|
||||
raw_metadata, compressed_metadata = metadata
|
||||
num_tokens = raw_metadata.num_actual_tokens
|
||||
hidden_states = hidden_states[:num_tokens]
|
||||
positions = positions[..., :num_tokens]
|
||||
if self.use_fused_pre_indexer:
|
||||
main_outputs: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
if attn.use_fused_qsa_prepare:
|
||||
if qkv is None or slot_mapping is None:
|
||||
raise ValueError("fused QSA prepare requires qkv and slot_mapping")
|
||||
projected_q, raw_keys = self.project(hidden_states)
|
||||
q = self._fused_pre_indexer(
|
||||
q, main_outputs = self._fused_prepare(
|
||||
projected_q,
|
||||
raw_keys,
|
||||
positions,
|
||||
raw_metadata,
|
||||
compressed_metadata,
|
||||
attn,
|
||||
qkv,
|
||||
slot_mapping,
|
||||
gate_out,
|
||||
)
|
||||
else:
|
||||
q, token_k = self.project_qk(hidden_states, positions)
|
||||
@@ -377,8 +413,8 @@ class QSAIndexer(nn.Module):
|
||||
if self.skip_topk:
|
||||
if out is None:
|
||||
raise RuntimeError("QSA top-k reuse requires an output buffer")
|
||||
return out
|
||||
return self._select(q, compressed_metadata, out)
|
||||
return out, main_outputs
|
||||
return self._select(q, compressed_metadata, out), main_outputs
|
||||
|
||||
|
||||
__all__ = ["QSAIndexer", "apply_qsa_rope"]
|
||||
|
||||
+156
-11
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused QSA pre-indexer kernel for the AMD Qwen4Exp path.
|
||||
"""Fused QSA prepare kernel for the AMD Qwen4Exp path.
|
||||
|
||||
Started as a copy of the NVIDIA kernel (``nvidia/ops/qsa_prepare.py``) and is
|
||||
kept separate so either vendor can change its own copy without re-validating
|
||||
@@ -119,6 +119,27 @@ def _norm_rope(
|
||||
return tl.reshape(result, (TILE_T, TILE_H, D))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _to_dst_dtype(x, dst, scale):
|
||||
"""Round to BF16 like the unfused path, then scale for an FP8 destination."""
|
||||
out_ty = dst.dtype.element_ty
|
||||
x = x.to(tl.bfloat16)
|
||||
if out_ty == tl.float8e4nv:
|
||||
x = x.to(tl.float32) / scale
|
||||
return x.to(out_ty)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_rotated(dst, y, o1, o2, scale):
|
||||
"""Store a normalized head whose first ``2 * len(o1)`` dims are rotated."""
|
||||
HALF: tl.constexpr = o1.shape[0]
|
||||
dims = tl.arange(0, y.shape[0])
|
||||
rot = tl.arange(0, HALF)
|
||||
tl.store(dst + dims, _to_dst_dtype(y, dst, scale), mask=dims >= 2 * HALF)
|
||||
tl.store(dst + rot, _to_dst_dtype(o1, dst, scale))
|
||||
tl.store(dst + HALF + rot, _to_dst_dtype(o2, dst, scale))
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"num_tokens",
|
||||
@@ -127,7 +148,7 @@ def _norm_rope(
|
||||
"num_k_work",
|
||||
]
|
||||
)
|
||||
def _qsa_pre_indexer_kernel(
|
||||
def _qsa_prepare_kernel(
|
||||
q_ptr,
|
||||
q_stride_token,
|
||||
k_ptr,
|
||||
@@ -171,8 +192,86 @@ def _qsa_pre_indexer_kernel(
|
||||
CACHE_HAS_ROPE_POS: tl.constexpr,
|
||||
MROPE_H: tl.constexpr,
|
||||
MROPE_W: tl.constexpr,
|
||||
main_qkv_ptr,
|
||||
main_q_norm_weight_ptr,
|
||||
main_k_norm_weight_ptr,
|
||||
main_eps,
|
||||
main_q_out_ptr,
|
||||
main_gate_out_ptr,
|
||||
main_cache_ptr,
|
||||
main_cache_stride_block,
|
||||
main_cache_stride_token,
|
||||
main_cache_stride_head,
|
||||
main_slots_ptr,
|
||||
main_k_scale,
|
||||
main_v_scale,
|
||||
MAIN_HQ: tl.constexpr,
|
||||
MAIN_HK: tl.constexpr,
|
||||
MAIN_D: tl.constexpr,
|
||||
MAIN_PAGE_SIZE: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
num_index_work = num_k_work + tl.cdiv(num_tokens, TILE_T_Q) * tl.cdiv(HQ, TILE_H_Q)
|
||||
if pid >= num_index_work:
|
||||
# Main attention: one program per (token, Q or KV head) after the
|
||||
# indexer work. RoPE covers the first D // 2 dims, the width of the
|
||||
# shared cos/sin table.
|
||||
main_pid = pid - num_index_work
|
||||
token = (main_pid // (MAIN_HQ + MAIN_HK)).to(tl.int64)
|
||||
head = main_pid % (MAIN_HQ + MAIN_HK)
|
||||
HALF: tl.constexpr = D // 4
|
||||
dims = tl.arange(0, MAIN_D)
|
||||
rot = tl.arange(0, HALF)
|
||||
row = main_qkv_ptr + token * (2 * (MAIN_HQ + MAIN_HK) * MAIN_D)
|
||||
is_k = head >= MAIN_HQ
|
||||
kv_head = head - MAIN_HQ
|
||||
if is_k:
|
||||
src = row + (2 * MAIN_HQ + kv_head) * MAIN_D
|
||||
weight_ptr = main_k_norm_weight_ptr
|
||||
else:
|
||||
src = row + 2 * head * MAIN_D
|
||||
weight_ptr = main_q_norm_weight_ptr
|
||||
x = tl.load(src + dims).to(tl.float32)
|
||||
inv_rms = tl.rsqrt(tl.sum(x * x, axis=0) / MAIN_D + main_eps)
|
||||
w = tl.load(weight_ptr + dims).to(tl.float32) + 1.0
|
||||
y = x * inv_rms * w
|
||||
x1 = tl.load(src + rot).to(tl.float32)
|
||||
x2 = tl.load(src + HALF + rot).to(tl.float32)
|
||||
w1 = tl.load(weight_ptr + rot).to(tl.float32) + 1.0
|
||||
w2 = tl.load(weight_ptr + HALF + rot).to(tl.float32) + 1.0
|
||||
x1 = (x1 * inv_rms * w1).to(tl.bfloat16).to(tl.float32)
|
||||
x2 = (x2 * inv_rms * w2).to(tl.bfloat16).to(tl.float32)
|
||||
pos = tl.load(pos_ptr + token * pos_stride_token).to(tl.int64)
|
||||
if IS_2D_POSITIONS:
|
||||
pos_h = tl.load(pos_ptr + pos_stride_axis + token * pos_stride_token)
|
||||
pos_w = tl.load(pos_ptr + 2 * pos_stride_axis + token * pos_stride_token)
|
||||
is_h = (rot % 3 == 1) & (rot < 3 * MROPE_H)
|
||||
is_w = (rot % 3 == 2) & (rot < 3 * MROPE_W)
|
||||
pos = tl.where(
|
||||
is_h, pos_h.to(tl.int64), tl.where(is_w, pos_w.to(tl.int64), pos)
|
||||
)
|
||||
cos = tl.load(cos_sin_ptr + pos * (D // 2) + rot).to(tl.float32)
|
||||
sin = tl.load(cos_sin_ptr + pos * (D // 2) + HALF + rot).to(tl.float32)
|
||||
o1 = x1 * cos - x2 * sin
|
||||
o2 = x2 * cos + x1 * sin
|
||||
if is_k:
|
||||
slot = tl.load(main_slots_ptr + token).to(tl.int64)
|
||||
if slot >= 0:
|
||||
dst = (
|
||||
main_cache_ptr
|
||||
+ (slot // MAIN_PAGE_SIZE) * main_cache_stride_block
|
||||
+ (slot % MAIN_PAGE_SIZE) * main_cache_stride_token
|
||||
+ kv_head * main_cache_stride_head
|
||||
)
|
||||
_store_rotated(dst, y, o1, o2, main_k_scale)
|
||||
v = tl.load(src + MAIN_HK * MAIN_D + dims)
|
||||
tl.store(dst + MAIN_D + dims, _to_dst_dtype(v, dst, main_v_scale))
|
||||
else:
|
||||
out = (token * MAIN_HQ + head) * MAIN_D
|
||||
_store_rotated(main_q_out_ptr + out, y, o1, o2, None)
|
||||
gate = tl.load(src + MAIN_D + dims)
|
||||
tl.store(main_gate_out_ptr + out + dims, gate)
|
||||
return
|
||||
# K work occupies the first programs; the remaining programs tile Q. This
|
||||
# keeps both paths in one launch while leaving their register shapes
|
||||
# independent.
|
||||
@@ -181,7 +280,7 @@ def _qsa_pre_indexer_kernel(
|
||||
num_head_tiles: tl.constexpr = tl.cdiv(HQ, TILE_H_Q)
|
||||
token_tile = q_pid // num_head_tiles
|
||||
head_tile = q_pid % num_head_tiles
|
||||
tokens = token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q)
|
||||
tokens = (token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q)).to(tl.int64)
|
||||
heads = head_tile * TILE_H_Q + tl.arange(0, TILE_H_Q)
|
||||
valid_tokens = tokens < num_tokens
|
||||
valid_heads = heads < HQ
|
||||
@@ -283,7 +382,8 @@ def _qsa_pre_indexer_kernel(
|
||||
& (source_tokens < num_tokens)
|
||||
)
|
||||
current_base = (
|
||||
k_ptr + tl.maximum(source_tokens, 0)[:, None] * k_stride_token
|
||||
k_ptr
|
||||
+ tl.maximum(source_tokens, 0).to(tl.int64)[:, None] * k_stride_token
|
||||
)
|
||||
cached_base = (
|
||||
state_cache_ptr
|
||||
@@ -397,7 +497,7 @@ def _qsa_pre_indexer_kernel(
|
||||
# One CTA per request commits only the suffix retained by the ring.
|
||||
num_state_rows = tl.minimum(query_len, STATE_SIZE)
|
||||
for state_offset in tl.range(0, num_state_rows):
|
||||
token = query_end - num_state_rows + state_offset
|
||||
token = (query_end - num_state_rows + state_offset).to(tl.int64)
|
||||
valid_token = (
|
||||
(token >= query_start) & (token < query_end) & (token < num_tokens)
|
||||
)
|
||||
@@ -447,7 +547,7 @@ def _qsa_pre_indexer_kernel(
|
||||
tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot)
|
||||
|
||||
|
||||
def qsa_pre_indexer(
|
||||
def qsa_prepare(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
@@ -468,11 +568,34 @@ def qsa_pre_indexer(
|
||||
compress_ratio: int,
|
||||
mrope_section: tuple[int, int, int] | None,
|
||||
rope_pos_offset: int | None,
|
||||
) -> None:
|
||||
"""Normalize Q, compress K, then update the circular raw state."""
|
||||
main_qkv: torch.Tensor,
|
||||
main_q_norm_weight: torch.Tensor,
|
||||
main_k_norm_weight: torch.Tensor,
|
||||
main_eps: float,
|
||||
main_kv_cache: torch.Tensor,
|
||||
main_slot_mapping: torch.Tensor,
|
||||
main_k_scale: float,
|
||||
main_v_scale: float,
|
||||
main_gate_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Normalize Q, compress K, then update the circular raw state.
|
||||
|
||||
Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write)
|
||||
and returns its Q and gate. ``main_gate_out`` lets the gate land in a
|
||||
caller-owned buffer, e.g. one that outlives an opaque custom op.
|
||||
"""
|
||||
num_tokens = q.shape[0]
|
||||
main_head_dim = main_kv_cache.shape[-1] // 2
|
||||
num_main_kv_heads = main_kv_cache.shape[2]
|
||||
num_main_q_heads = main_qkv.shape[1] // (2 * main_head_dim) - num_main_kv_heads
|
||||
main_q_out = main_qkv.new_empty(num_tokens, num_main_q_heads, main_head_dim)
|
||||
if main_gate_out is None:
|
||||
main_gate_out = torch.empty_like(main_q_out)
|
||||
else:
|
||||
assert main_gate_out.shape == main_q_out.shape
|
||||
assert main_gate_out.is_contiguous()
|
||||
if num_tokens == 0:
|
||||
return
|
||||
return main_q_out, main_gate_out
|
||||
num_q_heads, head_dim = q_out.shape[1:]
|
||||
assert cos_sin_cache.shape[-1] * 2 == head_dim
|
||||
assert q.shape == (num_tokens, num_q_heads * head_dim)
|
||||
@@ -497,6 +620,9 @@ def qsa_pre_indexer(
|
||||
pos_stride_axis, pos_stride_token = 0, positions.stride(0)
|
||||
section = mrope_section if mrope_section is not None else (0, 0, 0)
|
||||
assert len(section) == 3
|
||||
qkv_width = 2 * (num_main_q_heads + num_main_kv_heads) * main_head_dim
|
||||
assert main_qkv.shape == (num_tokens, qkv_width) and main_qkv.is_contiguous()
|
||||
assert main_slot_mapping.shape == (num_tokens,)
|
||||
|
||||
# Swept on MI355X (64-lane wavefronts) over TILE_T_Q 1-16, TILE_H_Q 1-4 and
|
||||
# num_warps 1/2/4, from 4 to 16384 tokens: no config beat these by more
|
||||
@@ -507,7 +633,8 @@ def qsa_pre_indexer(
|
||||
TILE_T_Q, TILE_H_Q = 2, 4
|
||||
num_k_work = k_work_metadata.shape[0]
|
||||
num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q)
|
||||
_qsa_pre_indexer_kernel[(num_k_work + num_q_work,)](
|
||||
num_main_work = num_tokens * (num_main_q_heads + num_main_kv_heads)
|
||||
_qsa_prepare_kernel[(num_k_work + num_q_work + num_main_work,)](
|
||||
q,
|
||||
q.stride(0),
|
||||
k,
|
||||
@@ -551,8 +678,26 @@ def qsa_pre_indexer(
|
||||
CACHE_HAS_ROPE_POS=cache_has_rope_pos,
|
||||
MROPE_H=section[1],
|
||||
MROPE_W=section[2],
|
||||
main_qkv_ptr=main_qkv,
|
||||
main_q_norm_weight_ptr=main_q_norm_weight,
|
||||
main_k_norm_weight_ptr=main_k_norm_weight,
|
||||
main_eps=main_eps,
|
||||
main_q_out_ptr=main_q_out,
|
||||
main_gate_out_ptr=main_gate_out,
|
||||
main_cache_ptr=main_kv_cache,
|
||||
main_cache_stride_block=main_kv_cache.stride(0),
|
||||
main_cache_stride_token=main_kv_cache.stride(1),
|
||||
main_cache_stride_head=main_kv_cache.stride(2),
|
||||
main_slots_ptr=main_slot_mapping,
|
||||
main_k_scale=main_k_scale,
|
||||
main_v_scale=main_v_scale,
|
||||
MAIN_HQ=num_main_q_heads,
|
||||
MAIN_HK=num_main_kv_heads,
|
||||
MAIN_D=main_head_dim,
|
||||
MAIN_PAGE_SIZE=main_kv_cache.shape[1],
|
||||
num_warps=1,
|
||||
)
|
||||
return main_q_out, main_gate_out
|
||||
|
||||
|
||||
__all__ = ["qsa_pre_indexer", "supports_fused_pre_indexer"]
|
||||
__all__ = ["qsa_prepare", "supports_fused_pre_indexer"]
|
||||
@@ -116,8 +116,8 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
key: torch.Tensor | None,
|
||||
value: torch.Tensor | None,
|
||||
kv_cache: torch.Tensor,
|
||||
attn_metadata: FlashAttentionMetadata,
|
||||
output: torch.Tensor,
|
||||
@@ -294,6 +294,11 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.indexer",
|
||||
)
|
||||
self.use_fused_qsa_prepare = (
|
||||
self.attn_output_gate
|
||||
and self.indexer.use_fused_pre_indexer
|
||||
and self.head_dim & (self.head_dim - 1) == 0
|
||||
)
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.register_buffer(
|
||||
"topk_indices_buffer",
|
||||
@@ -327,16 +332,22 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
query: torch.Tensor | None,
|
||||
key: torch.Tensor | None,
|
||||
value: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
qkv: torch.Tensor | None = None,
|
||||
gate: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
# query/key/value are None when the fused prepare runs inside the
|
||||
# indexer launch, which then writes the gate into ``gate``.
|
||||
metadata = get_forward_context().attn_metadata
|
||||
if isinstance(metadata, list):
|
||||
metadata = metadata[0]
|
||||
if not isinstance(metadata, dict):
|
||||
output.zero_()
|
||||
if gate is not None:
|
||||
gate.zero_()
|
||||
return
|
||||
main_metadata = cast(FlashAttentionMetadata, metadata[self.layer_name])
|
||||
if self.kv_cache.numel() == 0:
|
||||
@@ -349,10 +360,14 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
)
|
||||
if side_metadata.num_actual_tokens != num_tokens:
|
||||
raise RuntimeError("QSA main and side metadata token counts disagree")
|
||||
selected = self.indexer(
|
||||
selected, main_outputs = self.indexer(
|
||||
hidden_states,
|
||||
positions,
|
||||
self.topk_indices_buffer[:num_tokens],
|
||||
attn=self,
|
||||
qkv=qkv,
|
||||
slot_mapping=main_metadata.slot_mapping,
|
||||
gate_out=gate,
|
||||
)
|
||||
if selected.shape != (
|
||||
num_tokens,
|
||||
@@ -360,13 +375,22 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
):
|
||||
raise RuntimeError("QSA indexer returned an invalid selection shape")
|
||||
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
|
||||
impl.do_kv_cache_update(
|
||||
self,
|
||||
key,
|
||||
value,
|
||||
self.kv_cache,
|
||||
main_metadata.slot_mapping,
|
||||
)
|
||||
if main_outputs is None:
|
||||
assert key is not None and value is not None
|
||||
impl.do_kv_cache_update(
|
||||
self,
|
||||
key,
|
||||
value,
|
||||
self.kv_cache,
|
||||
main_metadata.slot_mapping,
|
||||
)
|
||||
else:
|
||||
query, _ = main_outputs
|
||||
# The gate is applied to every row outside this op; padded rows
|
||||
# must stay finite so they keep the zero output forward_qsa gives.
|
||||
assert gate is not None
|
||||
gate[num_tokens:].zero_()
|
||||
assert query is not None
|
||||
impl.forward_qsa(
|
||||
self,
|
||||
query,
|
||||
@@ -384,13 +408,40 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v, gate = self._project_qkv_gate(qkv, positions)
|
||||
num_tokens = hidden_states.shape[0]
|
||||
encoded_layer_name = _encode_layer_name(self.layer_name)
|
||||
if self.use_fused_qsa_prepare:
|
||||
# Norm/RoPE/gate and the K/V cache write happen inside the op.
|
||||
attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim)
|
||||
gate = torch.empty_like(attn_output)
|
||||
if current_platform.opaque_attention_op():
|
||||
torch.ops.vllm.qwen4_exp_qsa_prepare_with_output(
|
||||
hidden_states,
|
||||
positions,
|
||||
qkv,
|
||||
attn_output,
|
||||
gate,
|
||||
encoded_layer_name,
|
||||
)
|
||||
else:
|
||||
qwen4_exp_qsa_prepare_with_output(
|
||||
hidden_states,
|
||||
positions,
|
||||
qkv,
|
||||
attn_output,
|
||||
gate,
|
||||
encoded_layer_name,
|
||||
)
|
||||
flat_output = attn_output.view(num_tokens, -1)
|
||||
flat_output = flat_output * torch.sigmoid(gate.view(num_tokens, -1))
|
||||
output, _ = self.o_proj(flat_output)
|
||||
return output
|
||||
|
||||
q, k, v, gate = self._project_qkv_gate(qkv, positions)
|
||||
query = q.view(num_tokens, self.num_heads, self.head_dim)
|
||||
key = k.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
value = v.view(num_tokens, self.num_kv_heads, self.head_dim)
|
||||
attn_output = torch.empty_like(query)
|
||||
encoded_layer_name = _encode_layer_name(self.layer_name)
|
||||
if current_platform.opaque_attention_op():
|
||||
torch.ops.vllm.qwen4_exp_qsa_with_output(
|
||||
hidden_states,
|
||||
@@ -449,10 +500,43 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def qwen4_exp_qsa_prepare_with_output(
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
qkv: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
gate: torch.Tensor,
|
||||
layer_name: LayerNameType,
|
||||
) -> None:
|
||||
"""Run the QSA transaction with the main QK-norm/RoPE/gate fused in."""
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
layer = get_forward_context().no_compile_layers[layer_name]
|
||||
if not isinstance(layer, Qwen4ExpQSAAttention):
|
||||
raise TypeError(f"{layer_name} is not a Qwen4Exp QSA owner")
|
||||
layer._run_qsa(
|
||||
hidden_states,
|
||||
positions,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
output,
|
||||
qkv=qkv,
|
||||
gate=gate,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="qwen4_exp_qsa_prepare_with_output",
|
||||
op_func=qwen4_exp_qsa_prepare_with_output,
|
||||
mutates_args=["output", "gate"],
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"QSAIndexer",
|
||||
"Qwen4ExpQSAAttention",
|
||||
"Qwen4ExpQSAFlashAttentionBackend",
|
||||
"Qwen4ExpQSAFlashAttentionImpl",
|
||||
"qwen4_exp_qsa_prepare_with_output",
|
||||
"qwen4_exp_qsa_with_output",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user