diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index c2b2b054a3..865156f594 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -206,7 +206,7 @@ steps: depends_on: - image-build-amd commands: - - pytest -v -s models/qwen4_exp/test_qsa_amd.py + - pytest -v -s models/qwen4_exp/test_qsa_amd.py models/qwen4_exp/test_qsa_pre_indexer_amd.py # PLE and QSA import conflicting HC registrations in a shared process. - pytest -v -s models/qwen4_exp/test_ple.py diff --git a/tests/models/qwen4_exp/test_qsa_amd.py b/tests/models/qwen4_exp/test_qsa_amd.py index e8e39a7666..15c490223f 100644 --- a/tests/models/qwen4_exp/test_qsa_amd.py +++ b/tests/models/qwen4_exp/test_qsa_amd.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import inspect import math from types import SimpleNamespace from typing import Any @@ -9,15 +10,21 @@ import pytest import torch from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.models.qwen4_exp.amd import indexer_qsa from vllm.models.qwen4_exp.amd import ( model as _qwen4_exp_model, # noqa: F401 ) from vllm.models.qwen4_exp.amd import ple_layer as ple_layer_module from vllm.models.qwen4_exp.amd.indexer_qsa import ( + QSAIndexer, apply_qsa_rmsnorm, apply_qsa_rope, ) from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops +from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import ( + qsa_pre_indexer, + supports_fused_pre_indexer, +) from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON @@ -120,6 +127,149 @@ def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None: torch.testing.assert_close(output, norm.forward_native(tensor)) +def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer: + """A QSAIndexer whose unfused path runs for real, with no projection weights. + + Both branches have to be reachable: a fused-path test is only meaningful if + the unfused one it replaces would otherwise have executed. + """ + heads, head_dim = 2, 4 + indexer = QSAIndexer.__new__(QSAIndexer) + torch.nn.Module.__init__(indexer) + indexer.index_n_heads = heads + indexer.index_kv_heads = 1 + indexer.index_head_dim = head_dim + indexer.compress_ratio = 2 + indexer.use_fused_pre_indexer = use_fused + indexer.skip_topk = True + indexer.index_qk_proj = lambda hidden_states: ( + torch.zeros(hidden_states.shape[0], (heads + 1) * head_dim), + None, + ) + indexer.q_layernorm = GemmaRMSNorm(head_dim, eps=1e-6) + indexer.k_layernorm = GemmaRMSNorm(head_dim, eps=1e-6) + indexer.rotary_emb = SimpleNamespace( + cos_sin_cache=torch.zeros(16, head_dim), + rotary_dim=head_dim // 2, + apply_rotary_emb=lambda tensor, cos, sin: tensor, + _match_cos_sin_cache_dtype=lambda _: torch.zeros(16, head_dim), + ) + # The raw cache's full width carries the MRoPE position tail that the fused + # kernel writes; key_cache is only the leading key columns. + indexer.raw_key_cache = SimpleNamespace( + kv_cache=torch.zeros(1, 4, 1, head_dim + 12), + key_cache=torch.zeros(1, 4, 1, head_dim), + cache_rope_positions=True, + rope_position_offset=head_dim, + ) + indexer.compressed_key_cache = SimpleNamespace( + kv_cache=torch.zeros(1, 2, 1, head_dim) + ) + raw_metadata = SimpleNamespace( + num_actual_tokens=num_tokens, + slot_mapping=torch.arange(num_tokens), + block_table=torch.zeros(1, 1, dtype=torch.int32), + query_start_loc=torch.tensor([0, num_tokens], dtype=torch.int32), + logical_positions=torch.arange(num_tokens), + token_to_req=torch.zeros(num_tokens, dtype=torch.int32), + ) + compressed_metadata = SimpleNamespace( + num_actual_tokens=num_tokens, + slot_mapping=torch.arange(num_tokens), + k_work_metadata=torch.zeros(1, 2, dtype=torch.int32), + ) + indexer._metadata = lambda: (raw_metadata, compressed_metadata) + indexer.test_metadata = (raw_metadata, compressed_metadata) + return indexer + + +@pytest.mark.usefixtures("default_vllm_config") +def test_qsa_fused_pre_indexer_replaces_the_unfused_chain( + monkeypatch: pytest.MonkeyPatch, +) -> None: + indexer = _make_indexer(use_fused=True) + calls = [] + monkeypatch.setattr( + indexer_qsa, + "qsa_pre_indexer", + lambda *args, **kwargs: calls.append((args, kwargs)), + ) + monkeypatch.setattr( + QSAIndexer, + "_update_and_compress", + lambda *args, **kwargs: pytest.fail("fused path ran the unfused chain"), + ) + rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32) + + actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows) + + assert actual is rows + assert len(calls) == 1 + args, kwargs = calls[0] + passed = inspect.signature(qsa_pre_indexer).bind(*args, **kwargs).arguments + _, compressed_metadata = indexer.test_metadata + # The kernel addresses the raw cache at its full width and writes the MRoPE + # position tail itself, so it needs kv_cache, not the key_cache view. + assert passed["state_cache"] is indexer.raw_key_cache.kv_cache + assert passed["compressed_cache"] is indexer.compressed_key_cache.kv_cache + assert passed["k_work_metadata"] is compressed_metadata.k_work_metadata + assert passed["compress_ratio"] == indexer.compress_ratio + assert passed["rope_pos_offset"] == indexer.index_head_dim + + +@pytest.mark.usefixtures("default_vllm_config") +def test_qsa_unsupported_config_keeps_the_unfused_chain( + monkeypatch: pytest.MonkeyPatch, +) -> None: + indexer = _make_indexer(use_fused=False) + monkeypatch.setattr( + indexer_qsa, + "qsa_pre_indexer", + lambda *args, **kwargs: pytest.fail("unfused path ran the fused kernel"), + ) + updates = [] + monkeypatch.setattr( + QSAIndexer, + "_update_and_compress", + lambda *args, **kwargs: updates.append(args), + ) + rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32) + + actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows) + + assert actual is rows + assert len(updates) == 1 + + +@pytest.mark.parametrize( + ("overrides", "expected"), + [ + pytest.param({}, True, id="qwen3_8_flash_next"), + pytest.param({"is_neox_style": False}, False, id="not-neox"), + pytest.param({"rotary_dim": 128}, False, id="rotary-dim"), + pytest.param({"mrope_interleaved": False}, False, id="mrope-not-interleaved"), + pytest.param({"mrope_section": (11, 11, 10, 0)}, False, id="mrope-section-len"), + ], +) +def test_fused_pre_indexer_gate_is_config_shaped(overrides, expected) -> None: + # Qwen3.8-Flash-Next's indexer geometry, which both vendors satisfy. + rotary = SimpleNamespace( + rotary_dim=64, + is_neox_style=True, + mrope_section=(11, 11, 10), + mrope_interleaved=True, + ) + for name, value in overrides.items(): + setattr(rotary, name, value) + + assert ( + supports_fused_pre_indexer( + rotary, head_dim=128, num_kv_heads=1, compress_ratio=4 + ) + is expected + ) + + def test_qsa_selection_uses_portable_topk_on_rocm( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/models/qwen4_exp/test_qsa_pre_indexer_amd.py b/tests/models/qwen4_exp/test_qsa_pre_indexer_amd.py new file mode 100644 index 0000000000..8a39504e00 --- /dev/null +++ b/tests/models/qwen4_exp/test_qsa_pre_indexer_amd.py @@ -0,0 +1,344 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Correctness tests for the AMD fused QSA pre-indexer.""" + +import pytest +import torch + +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.models.qwen4_exp.amd.indexer_qsa import apply_qsa_rope +from vllm.models.qwen4_exp.amd.ops.qsa import ( + qsa_compress_groups_with_ratio, + qsa_store_cache_rows, +) +from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import ( + qsa_pre_indexer, +) +from vllm.models.qwen4_exp.common.qsa_cache import ( + canonical_qsa_rope_positions, + circular_qsa_slot_mapping, + compressed_qsa_slot_mapping, +) +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON + +requires_qsa_kernels = pytest.mark.skipif( + not current_platform.is_rocm() or not HAS_TRITON, + reason="the AMD QSA kernels require ROCm and Triton", +) + + +def _reference_gemma_rmsnorm( + x: torch.Tensor, + weight: torch.Tensor, + eps: float, +) -> torch.Tensor: + """The unfused Gemma RMSNorm the AMD reference path actually runs.""" + norm = GemmaRMSNorm(weight.shape[-1], eps=eps).to( + device=weight.device, dtype=weight.dtype + ) + with torch.no_grad(): + norm.weight.copy_(weight) + return norm.forward_native(x) + + +HQ, D = 4, 128 +CR = 4 +MROPE_SECTION = (11, 11, 10) +EPS = 1e-6 +BLOCK_SIZE = 16 +COMP_PAGE = BLOCK_SIZE // CR +ROPE_POS_OFFSET = D +RTOL = 1.6e-2 +ATOL = 1e-2 +MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8]) + + +def _make_block_table(block_counts): + num_blocks = sum(block_counts) + table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32) + physical_blocks = torch.randperm(num_blocks) + offset = 0 + for request, count in enumerate(block_counts): + table[request, :count] = physical_blocks[offset : offset + count] + offset += count + return table, num_blocks + + +def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> None: + # e4m3 is sign-magnitude, so within a sign the uint8 code order matches the + # value order and one ulp is one code step. The two paths' intermediates + # differ in the pooling accumulation order, which at denormal magnitudes + # (absolute grid step 2^-9) shows up as up to 2 code steps. + code_diff = ( + actual.view(torch.uint8).int() - expected.view(torch.uint8).int() + ).abs() + abs_diff = (actual.float() - expected.float()).abs() + assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all()) + + +@requires_qsa_kernels +@pytest.mark.usefixtures("default_vllm_config") +@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize( + "mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens", + [ + pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"), + pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"), + pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"), + pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"), + pytest.param( + True, + False, + False, + 8, + *MIXED_BATCH, + id="mrope-model-1d-no-position-cache", + ), + pytest.param( + False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache" + ), + pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"), + pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"), + ], +) +def test_qsa_fused_pre_indexer_matches_unfused( + indexer_dtype, + mrope, + is_2d_positions, + cache_rope_positions, + state_size, + seq_lens, + query_lens, + history_lens, +) -> None: + from vllm.model_executor.layers.rotary_embedding import get_rope + + if indexer_dtype == torch.float8_e4m3fn: + pytest.skip("the AMD QSA indexer cache is bf16-only") + + # The FP8 comparison below is a one-ulp bound on data this test generates, + # so an unseeded run decides for itself whether it reproduces a rounding + # difference. Seed it, or the kernel's bit-exactness is only sampled. + torch.manual_seed(0) + + device = "cuda" + rope_params = { + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default", + } + if mrope: + rope_params["mrope_interleaved"] = True + rope_params["mrope_section"] = list(MROPE_SECTION) + with torch.device(device): + rope = get_rope( + head_size=256, + max_position=32768, + rope_parameters=rope_params, + dtype=torch.bfloat16, + ) + + token_to_req = torch.cat( + [ + torch.full((length,), request, dtype=torch.int32) + for request, length in enumerate(query_lens) + ] + ).to(device) + logical_positions = torch.cat( + [ + torch.arange(seq_len - query_len, seq_len, dtype=torch.int64) + for seq_len, query_len in zip(seq_lens, query_lens) + ] + ).to(device) + num_tokens = logical_positions.numel() + query_start_loc = torch.tensor( + [0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32 + ).to(device) + positions = ( + torch.stack( + [ + logical_positions, + logical_positions // 7 + 3, + logical_positions // 13 + 11, + ] + ) + if is_2d_positions + else logical_positions + ) + position_rows = ( + canonical_qsa_rope_positions(positions) + if cache_rope_positions + else logical_positions.view(-1, 1, 1).expand(-1, 1, 3) + ) + + compressed_block_counts = [ + (seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens + ] + raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens)) + compressed_block_table, num_compressed_blocks = _make_block_table( + compressed_block_counts + ) + raw_block_table = raw_block_table.to(device) + raw_slots = circular_qsa_slot_mapping( + raw_block_table, + token_to_req, + logical_positions, + state_size, + query_start_loc, + ) + compressed_slots = compressed_qsa_slot_mapping( + compressed_block_table.to(device), + token_to_req, + logical_positions, + COMP_PAGE, + CR, + ) + group_counts = torch.tensor( + [ + seq_len // CR - (seq_len - query_len) // CR + for seq_len, query_len in zip(seq_lens, query_lens) + ], + dtype=torch.int32, + device=device, + ) + k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts)) + k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)]) + work_requests = torch.repeat_interleave( + torch.arange(len(query_lens), dtype=torch.int32, device=device), + k_work_counts, + ) + local_work = torch.arange( + int(k_start_loc[-1]), dtype=torch.int32, device=device + ) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts) + max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR + k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device) + k_work_metadata[: work_requests.numel()] = torch.stack( + (work_requests, local_work), dim=1 + ) + + raw_width = D + 12 if cache_rope_positions else D + # Match vLLM's padded-page cache layout: rows are contiguous, while physical + # blocks have a larger stride than their logical contents. + raw_page_elements = state_size * raw_width + fused_raw_storage = torch.zeros( + num_raw_blocks, + raw_page_elements + 16, + dtype=torch.bfloat16, + device=device, + ) + fused_raw = torch.as_strided( + fused_raw_storage, + (num_raw_blocks, state_size, 1, raw_width), + (raw_page_elements + 16, raw_width, raw_width, 1), + ) + for request, history_len in enumerate(history_lens): + history_end = seq_lens[request] - query_lens[request] + for position in range(history_end - history_len, history_end): + block = int(raw_block_table[request, 0]) + row = fused_raw[block, position % state_size, 0] + row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device) + if cache_rope_positions: + row[ROPE_POS_OFFSET:].view(torch.int64).copy_( + torch.tensor( + [position, position // 7 + 3, position // 13 + 11], + dtype=torch.int64, + device=device, + ) + ) + unfused_raw = fused_raw.clone() + compressed_page_elements = COMP_PAGE * D + fused_compressed_storage = torch.zeros( + num_compressed_blocks, + compressed_page_elements + 16, + dtype=indexer_dtype, + device=device, + ) + fused_compressed = torch.as_strided( + fused_compressed_storage, + (num_compressed_blocks, COMP_PAGE, 1, D), + (compressed_page_elements + 16, D, D, 1), + ) + unfused_compressed = fused_compressed.clone() + + projected_qk = torch.randn( + num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device + ) + q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2 + k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2 + + fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device) + qsa_pre_indexer( + projected_qk[:, : HQ * D], + projected_qk[:, HQ * D :], + positions, + rope.cos_sin_cache, + q_weight, + k_weight, + EPS, + fused_query, + fused_raw, + raw_slots, + raw_block_table, + query_start_loc, + logical_positions, + fused_compressed, + compressed_slots, + k_work_metadata, + compress_ratio=CR, + mrope_section=MROPE_SECTION if mrope else None, + rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None, + ) + + unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D) + unfused_query = _reference_gemma_rmsnorm( + unfused_query.reshape(-1, D), q_weight, EPS + ).reshape_as(unfused_query) + unfused_query = apply_qsa_rope(rope, positions, unfused_query) + + raw_keys = unfused_raw[..., :D] + rope_positions = ( + unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64) + if cache_rope_positions + else None + ) + pooled, first_positions = qsa_compress_groups_with_ratio( + projected_qk[:, HQ * D :].reshape(-1, 1, D), + position_rows, + raw_keys, + raw_block_table, + token_to_req, + query_start_loc, + logical_positions, + compressed_slots, + CR, + rope_positions, + ) + compressed_rows = _reference_gemma_rmsnorm( + pooled.reshape(-1, D), k_weight, EPS + ).reshape(-1, 1, D) + group_positions = ( + first_positions.transpose(0, 1) if mrope else first_positions[:, 0] + ) + compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows) + qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows) + qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :]) + if rope_positions is not None: + qsa_store_cache_rows(rope_positions, raw_slots, position_rows) + + if indexer_dtype == torch.float8_e4m3fn: + # Both paths round the same ~bf16 intermediates to e4m3. Bitwise + # equality (the dsv4 indexer test's bar) does not hold here: the 1D + # reference rope (forward_cuda) differs from _norm_rope by up to 1 + # bf16 ulp, and even the MRoPE pair flips a code occasionally. + unfused_query = unfused_query.to(indexer_dtype) + assert_fp8_within_one_ulp(fused_query, unfused_query) + else: + torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL) + assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16)) + if indexer_dtype == torch.float8_e4m3fn: + assert_fp8_within_one_ulp(fused_compressed, unfused_compressed) + else: + torch.testing.assert_close( + fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL + ) diff --git a/vllm/models/qwen4_exp/amd/indexer_qsa.py b/vllm/models/qwen4_exp/amd/indexer_qsa.py index 4345763fd4..162914f39e 100644 --- a/vllm/models/qwen4_exp/amd/indexer_qsa.py +++ b/vllm/models/qwen4_exp/amd/indexer_qsa.py @@ -23,6 +23,7 @@ from ..common.qsa_cache import ( QSAKeyStateCache, canonical_qsa_rope_positions, ) +from .ops.qsa_pre_indexer import qsa_pre_indexer, supports_fused_pre_indexer def apply_qsa_rope( @@ -98,6 +99,12 @@ class QSAIndexer(nn.Module): self.token_topk = int(config.indexer_budget) self.compress_ratio = int(config.indexer_compress_ratio) self.rotary_emb = rotary_emb + self.use_fused_pre_indexer = supports_fused_pre_indexer( + rotary_emb, + self.index_head_dim, + self.index_kv_heads, + self.compress_ratio, + ) self.prefix = prefix # MTP step 0 selects the target-aligned rows; later steps reuse them # while continuing to update the QSA side cache. @@ -143,20 +150,27 @@ class QSAIndexer(nn.Module): def output_width(self) -> int: return self.token_topk + self.compress_ratio - 1 - def project_qk( + def project( self, hidden_states: torch.Tensor, - positions: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - """Project replicated Q/K, normalize+rotate Q, and preserve raw K.""" + """Project replicated Q/K and split, leaving both operands unrotated.""" qk, _ = self.index_qk_proj(hidden_states) - q_raw, token_k = qk.split( + return qk.split( ( self.index_n_heads * self.index_head_dim, self.index_kv_heads * self.index_head_dim, ), dim=-1, ) + + def project_qk( + self, + hidden_states: torch.Tensor, + positions: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Project replicated Q/K, normalize+rotate Q, and preserve raw K.""" + q_raw, token_k = self.project(hidden_states) q = q_raw.reshape(-1, self.index_n_heads, self.index_head_dim) q = apply_qsa_rmsnorm( self.q_layernorm, @@ -251,6 +265,52 @@ class QSAIndexer(nn.Module): position_rows, ) + def _fused_pre_indexer( + self, + projected_q: torch.Tensor, + raw_keys: torch.Tensor, + positions: torch.Tensor, + raw_metadata: QSAForwardMetadata, + compressed_metadata: QSAForwardMetadata, + ) -> torch.Tensor: + """Normalize and rotate Q, compress K, and write both caches in one launch. + + Replaces ``project_qk``'s norm/RoPE half and the whole of + ``_update_and_compress``; the two must stay numerically interchangeable. + """ + raw_key_cache = self.raw_key_cache + q = projected_q.new_empty( + raw_metadata.num_actual_tokens, + self.index_n_heads, + self.index_head_dim, + ) + qsa_pre_indexer( + projected_q, + raw_keys, + positions, + self.rotary_emb.cos_sin_cache, + self.q_layernorm.weight, + self.k_layernorm.weight, + self.q_layernorm.variance_epsilon, + q, + raw_key_cache.kv_cache, + raw_metadata.slot_mapping, + raw_metadata.block_table, + raw_metadata.query_start_loc, + raw_metadata.logical_positions, + self.compressed_key_cache.kv_cache, + compressed_metadata.slot_mapping, + compressed_metadata.k_work_metadata, + compress_ratio=self.compress_ratio, + mrope_section=getattr(self.rotary_emb, "mrope_section", None), + rope_pos_offset=( + raw_key_cache.rope_position_offset + if raw_key_cache.cache_rope_positions + else None + ), + ) + return q + def _select( self, q: torch.Tensor, @@ -295,15 +355,25 @@ class QSAIndexer(nn.Module): return result raw_metadata, compressed_metadata = metadata num_tokens = raw_metadata.num_actual_tokens - q, token_k = self.project_qk( - hidden_states[:num_tokens], positions[..., :num_tokens] - ) - self._update_and_compress( - token_k, - positions[..., :num_tokens], - raw_metadata, - compressed_metadata, - ) + hidden_states = hidden_states[:num_tokens] + positions = positions[..., :num_tokens] + if self.use_fused_pre_indexer: + projected_q, raw_keys = self.project(hidden_states) + q = self._fused_pre_indexer( + projected_q, + raw_keys, + positions, + raw_metadata, + compressed_metadata, + ) + else: + q, token_k = self.project_qk(hidden_states, positions) + self._update_and_compress( + token_k, + positions, + raw_metadata, + compressed_metadata, + ) if self.skip_topk: if out is None: raise RuntimeError("QSA top-k reuse requires an output buffer") diff --git a/vllm/models/qwen4_exp/amd/ops/qsa_pre_indexer.py b/vllm/models/qwen4_exp/amd/ops/qsa_pre_indexer.py new file mode 100644 index 0000000000..6a4873882f --- /dev/null +++ b/vllm/models/qwen4_exp/amd/ops/qsa_pre_indexer.py @@ -0,0 +1,558 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fused QSA pre-indexer kernel for the AMD Qwen4Exp path. + +Started as a copy of the NVIDIA kernel (``nvidia/ops/qsa_prepare.py``) and is +kept separate so either vendor can change its own copy without re-validating +the other. The state source select is two masked loads rather than a pointer +select, which ROCm Triton rejects. +""" + +import torch +from torch import nn + +from vllm.triton_utils import tl, triton + + +def supports_fused_pre_indexer( + rotary_emb: nn.Module, + head_dim: int, + num_kv_heads: int, + compress_ratio: int, +) -> bool: + """Report whether this indexer's shapes match the fused kernel's assumptions. + + The kernel hard-codes the rotary layout and the single-KV-head group + compression it was written for; everything it rejects has a working unfused + path. + """ + rotary_dim = int(rotary_emb.rotary_dim) + mrope_section = getattr(rotary_emb, "mrope_section", None) + return ( + bool(getattr(rotary_emb, "is_neox_style", False)) + and ( + not mrope_section + or ( + len(mrope_section) == 3 + and sum(mrope_section) == rotary_dim // 2 + and bool(getattr(rotary_emb, "mrope_interleaved", False)) + ) + ) + and head_dim == 128 + and rotary_dim == 64 + and num_kv_heads == 1 + and compress_ratio > 1 + and compress_ratio & (compress_ratio - 1) == 0 + ) + + +@triton.jit +def _norm_rope( + x, + pos_t, + pos_h, + pos_w, + cos_sin_ptr, + cos_sin_stride, + norm_weight_ptr, + eps, + IS_MROPE: tl.constexpr, + MROPE_H: tl.constexpr, + MROPE_W: tl.constexpr, +): + """Apply Gemma RMSNorm and selected-axis NeoX RoPE to register rows.""" + TILE_T: tl.constexpr = x.shape[0] + TILE_H: tl.constexpr = x.shape[1] + D: tl.constexpr = x.shape[2] + ROWS: tl.constexpr = TILE_T * TILE_H + HALF: tl.constexpr = D // 2 + QUARTER: tl.constexpr = D // 4 + pairs = tl.arange(0, QUARTER) + if IS_MROPE: + # Qwen interleaves temporal, height, and width rotary pairs. Each axis + # still indexes the same position-major cos/sin table. + h_mask = ((pairs % 3) == 1) & (pairs <= 3 * MROPE_H) + w_mask = ((pairs % 3) == 2) & (pairs <= 3 * MROPE_W) + t_mask = ~(h_mask | w_mask) + base = cos_sin_ptr + pairs[None, :] + pos_rows = (pos_t, pos_h, pos_w) + axis_masks = (t_mask, h_mask, w_mask) + cos = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty) + sin = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty) + for axis in tl.static_range(3): + cos += tl.load( + base + pos_rows[axis][:, None] * cos_sin_stride, + mask=axis_masks[axis][None, :], + other=0, + ) + sin += tl.load( + base + pos_rows[axis][:, None] * cos_sin_stride + QUARTER, + mask=axis_masks[axis][None, :], + other=0, + ) + else: + cos = tl.load(cos_sin_ptr + pos_t[:, None] * cos_sin_stride + pairs[None, :]) + sin = tl.load( + cos_sin_ptr + pos_t[:, None] * cos_sin_stride + QUARTER + pairs[None, :] + ) + + cos = tl.reshape( + tl.broadcast_to(cos[:, None, :], (TILE_T, TILE_H, QUARTER)), + (ROWS, QUARTER), + ) + sin = tl.reshape( + tl.broadcast_to(sin[:, None, :], (TILE_T, TILE_H, QUARTER)), + (ROWS, QUARTER), + ) + x = tl.reshape(x, (ROWS, D)).to(tl.float32) + weight = tl.load(norm_weight_ptr + tl.arange(0, D)).to(tl.float32) + 1.0 + rrms = tl.rsqrt(tl.sum(x * x, axis=1) / D + eps) + y = (x * rrms[:, None] * weight[None, :]).to(cos.dtype) + rotated, passthrough = tl.split( + tl.permute(tl.reshape(y, (ROWS, 2, HALF)), (0, 2, 1)) + ) + r0, r1 = tl.split(tl.permute(tl.reshape(rotated, (ROWS, 2, QUARTER)), (0, 2, 1))) + out0 = r0 * cos - r1 * sin + out1 = r1 * cos + r0 * sin + rotated = tl.reshape(tl.permute(tl.join(out0, out1), (0, 2, 1)), (ROWS, HALF)) + result = tl.reshape(tl.permute(tl.join(rotated, passthrough), (0, 2, 1)), (ROWS, D)) + return tl.reshape(result, (TILE_T, TILE_H, D)) + + +@triton.jit( + do_not_specialize=[ + "num_tokens", + "num_state_blocks", + "num_compressed_blocks", + "num_k_work", + ] +) +def _qsa_pre_indexer_kernel( + q_ptr, + q_stride_token, + k_ptr, + k_stride_token, + pos_ptr, + pos_stride_axis, + pos_stride_token, + cos_sin_ptr, + q_norm_weight_ptr, + k_norm_weight_ptr, + eps, + q_out_ptr, + q_out_stride_token, + q_out_stride_head, + state_cache_ptr, + state_cache_stride_block, + state_cache_stride_token, + state_slots_ptr, + state_table_ptr, + state_table_stride_req, + query_start_loc_ptr, + logical_positions_ptr, + compressed_slots_ptr, + k_work_metadata_ptr, + compressed_cache_ptr, + compressed_cache_stride_block, + compressed_cache_stride_token, + num_tokens, + num_state_blocks, + num_compressed_blocks, + num_k_work, + HQ: tl.constexpr, + D: tl.constexpr, + TILE_T_Q: tl.constexpr, + TILE_H_Q: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + STATE_SIZE: tl.constexpr, + COMP_PAGE_SIZE: tl.constexpr, + IS_2D_POSITIONS: tl.constexpr, + IS_K_MROPE: tl.constexpr, + CACHE_HAS_ROPE_POS: tl.constexpr, + MROPE_H: tl.constexpr, + MROPE_W: tl.constexpr, +): + pid = tl.program_id(0) + # K work occupies the first programs; the remaining programs tile Q. This + # keeps both paths in one launch while leaving their register shapes + # independent. + if pid >= num_k_work: + q_pid = pid - num_k_work + num_head_tiles: tl.constexpr = tl.cdiv(HQ, TILE_H_Q) + token_tile = q_pid // num_head_tiles + head_tile = q_pid % num_head_tiles + tokens = token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q) + heads = head_tile * TILE_H_Q + tl.arange(0, TILE_H_Q) + valid_tokens = tokens < num_tokens + valid_heads = heads < HQ + dims = tl.arange(0, D) + mask = valid_tokens[:, None, None] & valid_heads[None, :, None] + x = tl.load( + q_ptr + + tokens[:, None, None] * q_stride_token + + heads[None, :, None] * D + + dims[None, None, :], + mask=mask, + other=0.0, + ) + pos_t = tl.load(pos_ptr + tokens * pos_stride_token, mask=valid_tokens, other=0) + if IS_2D_POSITIONS: + pos_h = tl.load( + pos_ptr + pos_stride_axis + tokens * pos_stride_token, + mask=valid_tokens, + other=0, + ) + pos_w = tl.load( + pos_ptr + 2 * pos_stride_axis + tokens * pos_stride_token, + mask=valid_tokens, + other=0, + ) + else: + pos_h = pos_t + pos_w = pos_t + y = _norm_rope( + x, + pos_t, + pos_h, + pos_w, + cos_sin_ptr, + D // 2, + q_norm_weight_ptr, + eps, + IS_2D_POSITIONS, + MROPE_H, + MROPE_W, + ) + tl.store( + q_out_ptr + + tokens[:, None, None] * q_out_stride_token + + heads[None, :, None] * q_out_stride_head + + dims[None, None, :], + y, + mask=mask, + ) + + if pid < num_k_work: + # One work item owns one completed compression group. Work item zero + # additionally commits the request's current raw-K suffix below. + work_metadata = tl.load(k_work_metadata_ptr + pid * 2 + tl.arange(0, 2)) + request, work_in_request = tl.split(work_metadata) + if request < 0: + return + + query_start = tl.load(query_start_loc_ptr + request) + query_end = tl.load(query_start_loc_ptr + request + 1) + query_len = query_end - query_start + chunk_end = tl.load(logical_positions_ptr + query_end - 1) + chunk_start = chunk_end - query_len + 1 + num_groups = (chunk_end + 1) // COMPRESS_RATIO - chunk_start // COMPRESS_RATIO + dims = tl.arange(0, D) + + if work_in_request < num_groups: + first_boundary = ( + (chunk_start + COMPRESS_RATIO) // COMPRESS_RATIO + ) * COMPRESS_RATIO - 1 + end_position = first_boundary + work_in_request * COMPRESS_RATIO + boundary_token = query_start + end_position - chunk_start + valid_token = ( + (boundary_token >= query_start) + & (boundary_token < query_end) + & (boundary_token < num_tokens) + ) + compressed_slot = tl.load( + compressed_slots_ptr + boundary_token, + mask=valid_token, + other=-1, + ) + valid = ( + valid_token + & (compressed_slot >= 0) + & (compressed_slot < num_compressed_blocks * COMP_PAGE_SIZE) + ) + state_block = tl.load(state_table_ptr + request * state_table_stride_req) + state_block_valid = (state_block >= 0) & (state_block < num_state_blocks) + safe_state_block = tl.maximum(state_block, 0).to(tl.int64) + group_offsets = tl.arange(0, COMPRESS_RATIO) + source_positions = end_position - (COMPRESS_RATIO - 1) + group_offsets + source_in_chunk = source_positions >= chunk_start + source_in_cache = source_positions < chunk_start + source_tokens = query_start + source_positions - chunk_start + source_tokens_valid = ( + (source_tokens >= query_start) + & (source_tokens < query_end) + & (source_tokens < num_tokens) + ) + current_base = ( + k_ptr + tl.maximum(source_tokens, 0)[:, None] * k_stride_token + ) + cached_base = ( + state_cache_ptr + + safe_state_block * state_cache_stride_block + + (source_positions % STATE_SIZE)[:, None] * state_cache_stride_token + ) + # Only the first completed group can cross the chunk boundary, so + # every row comes from exactly one of the two sources. Selecting + # between the base pointers would need a pointer-typed tl.where, + # which ROCm's Triton rejects ("'tt.addptr' op failed to verify + # that result type matches ptr type"), so select the loaded values + # instead. The two masks are complementary, so no lane fetches from + # both and memory traffic is unchanged. + # + # Both bases still need the alignment stated explicitly. Without it + # Triton vectorizes these loads differently, which reorders the + # pooling reduction below; float addition is not associative, so the + # compressed rows stop matching the unfused path bit for bit and an + # FP8 cache rounds a near-zero pooled value to the wrong code. + current_base = tl.multiple_of(current_base, (8, 8)) + cached_base = tl.multiple_of(cached_base, (8, 8)) + from_chunk = valid & source_in_chunk[:, None] & source_tokens_valid[:, None] + from_cache = valid & source_in_cache[:, None] & state_block_valid + current = tl.load(current_base + dims[None, :], mask=from_chunk, other=0.0) + cached = tl.load(cached_base + dims[None, :], mask=from_cache, other=0.0) + source = tl.where(source_in_chunk[:, None], current, cached).to(tl.float32) + # Match the unfused path's BF16 pooled tensor before RMSNorm. + pooled = ( + (tl.sum(source, axis=0) / COMPRESS_RATIO).to(tl.bfloat16).to(tl.float32) + ) + + first_position = end_position - (COMPRESS_RATIO - 1) + if CACHE_HAS_ROPE_POS: + # RoPE uses the first token in the pooled group. Its exact MRoPE + # coordinates may live in this chunk or the raw-state ring. + first_in_chunk = first_position >= chunk_start + first_token = query_start + first_position - chunk_start + first_token_valid = ( + (first_token >= query_start) + & (first_token < query_end) + & (first_token < num_tokens) + ) + safe_first_token = tl.maximum(first_token, 0) + load_current_position = first_in_chunk & first_token_valid + first_pos_t = tl.load( + pos_ptr + safe_first_token * pos_stride_token, + mask=load_current_position, + other=0, + ) + if IS_2D_POSITIONS: + first_pos_h = tl.load( + pos_ptr + pos_stride_axis + safe_first_token * pos_stride_token, + mask=load_current_position, + other=0, + ) + first_pos_w = tl.load( + pos_ptr + + 2 * pos_stride_axis + + safe_first_token * pos_stride_token, + mask=load_current_position, + other=0, + ) + else: + first_pos_h = first_pos_t + first_pos_w = first_pos_t + tail = ( + state_cache_ptr + + safe_state_block * state_cache_stride_block + + (first_position % STATE_SIZE) * state_cache_stride_token + + D + ).to(tl.pointer_type(tl.int64)) + load_cached_position = ~first_in_chunk & state_block_valid + cached_pos_t = tl.load(tail, mask=load_cached_position, other=0) + cached_pos_h = tl.load(tail + 1, mask=load_cached_position, other=0) + cached_pos_w = tl.load(tail + 2, mask=load_cached_position, other=0) + pos_t = tl.where(first_in_chunk, first_pos_t.to(tl.int64), cached_pos_t) + pos_h = tl.where(first_in_chunk, first_pos_h.to(tl.int64), cached_pos_h) + pos_w = tl.where(first_in_chunk, first_pos_w.to(tl.int64), cached_pos_w) + else: + pos_t = first_position + pos_h = first_position + pos_w = first_position + y = _norm_rope( + tl.reshape(pooled, (1, 1, D)), + pos_t + tl.arange(0, 1), + pos_h + tl.arange(0, 1), + pos_w + tl.arange(0, 1), + cos_sin_ptr, + D // 2, + k_norm_weight_ptr, + eps, + IS_K_MROPE, + MROPE_H, + MROPE_W, + ) + compressed_block = (compressed_slot // COMP_PAGE_SIZE).to(tl.int64) + compressed_row = compressed_slot % COMP_PAGE_SIZE + tl.store( + compressed_cache_ptr + + compressed_block * compressed_cache_stride_block + + compressed_row * compressed_cache_stride_token + + dims, + tl.reshape(y, (D,)), + mask=valid, + ) + + if work_in_request == 0: + # This CTA may have just read historical rows from the circular buffer. + # Keep every lane past those loads before overwriting the same ring. + tl.debug_barrier() + # One CTA per request commits only the suffix retained by the ring. + num_state_rows = tl.minimum(query_len, STATE_SIZE) + for state_offset in tl.range(0, num_state_rows): + token = query_end - num_state_rows + state_offset + valid_token = ( + (token >= query_start) & (token < query_end) & (token < num_tokens) + ) + slot = tl.load(state_slots_ptr + token, mask=valid_token, other=-1) + valid_slot = ( + valid_token & (slot >= 0) & (slot < num_state_blocks * STATE_SIZE) + ) + safe_slot = tl.maximum(slot, 0) + state_row = ( + state_cache_ptr + + (safe_slot // STATE_SIZE).to(tl.int64) * state_cache_stride_block + + (safe_slot % STATE_SIZE) * state_cache_stride_token + ) + k = tl.load( + k_ptr + tl.maximum(token, 0) * k_stride_token + dims, + mask=valid_slot, + other=0.0, + ) + tl.store(state_row + dims, k, mask=valid_slot) + if CACHE_HAS_ROPE_POS: + pos_t = tl.load( + pos_ptr + tl.maximum(token, 0) * pos_stride_token, + mask=valid_slot, + other=0, + ) + if IS_2D_POSITIONS: + pos_h = tl.load( + pos_ptr + + pos_stride_axis + + tl.maximum(token, 0) * pos_stride_token, + mask=valid_slot, + other=0, + ) + pos_w = tl.load( + pos_ptr + + 2 * pos_stride_axis + + tl.maximum(token, 0) * pos_stride_token, + mask=valid_slot, + other=0, + ) + else: + pos_h = pos_t + pos_w = pos_t + tail = (state_row + D).to(tl.pointer_type(tl.int64)) + tl.store(tail, pos_t.to(tl.int64), mask=valid_slot) + tl.store(tail + 1, pos_h.to(tl.int64), mask=valid_slot) + tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot) + + +def qsa_pre_indexer( + q: torch.Tensor, + k: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + eps: float, + q_out: torch.Tensor, + state_cache: torch.Tensor, + state_slots: torch.Tensor, + state_block_table: torch.Tensor, + query_start_loc: torch.Tensor, + logical_positions: torch.Tensor, + compressed_cache: torch.Tensor, + compressed_slots: torch.Tensor, + k_work_metadata: torch.Tensor, + *, + compress_ratio: int, + mrope_section: tuple[int, int, int] | None, + rope_pos_offset: int | None, +) -> None: + """Normalize Q, compress K, then update the circular raw state.""" + num_tokens = q.shape[0] + if num_tokens == 0: + return + num_q_heads, head_dim = q_out.shape[1:] + assert cos_sin_cache.shape[-1] * 2 == head_dim + assert q.shape == (num_tokens, num_q_heads * head_dim) + assert k.shape == (num_tokens, head_dim) + assert q.stride(-1) == 1 + assert k.stride(-1) == 1 + assert q_out.stride(-1) == 1 + assert cos_sin_cache.is_contiguous() + assert state_cache.stride(-1) == 1 + assert compressed_cache.stride(-1) == 1 + assert k_work_metadata.ndim == 2 and k_work_metadata.shape[1] == 2 + is_2d_positions = positions.ndim == 2 + is_k_mrope = bool(mrope_section) + cache_has_rope_pos = rope_pos_offset is not None + assert rope_pos_offset is None or rope_pos_offset == head_dim + if is_2d_positions: + assert positions.shape == (3, num_tokens) + assert is_k_mrope + pos_stride_axis, pos_stride_token = positions.stride() + else: + assert positions.shape == (num_tokens,) + pos_stride_axis, pos_stride_token = 0, positions.stride(0) + section = mrope_section if mrope_section is not None else (0, 0, 0) + assert len(section) == 3 + + # Swept on MI355X (64-lane wavefronts) over TILE_T_Q 1-16, TILE_H_Q 1-4 and + # num_warps 1/2/4, from 4 to 16384 tokens: no config beat these by more + # than noise, and 2 or 4 warps were slower everywhere. + if num_tokens <= 4096: + TILE_T_Q, TILE_H_Q = 2, 2 + else: + TILE_T_Q, TILE_H_Q = 2, 4 + num_k_work = k_work_metadata.shape[0] + num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q) + _qsa_pre_indexer_kernel[(num_k_work + num_q_work,)]( + q, + q.stride(0), + k, + k.stride(0), + positions, + pos_stride_axis, + pos_stride_token, + cos_sin_cache, + q_norm_weight, + k_norm_weight, + eps, + q_out, + q_out.stride(0), + q_out.stride(1), + state_cache, + state_cache.stride(0), + state_cache.stride(1), + state_slots, + state_block_table, + state_block_table.stride(0), + query_start_loc, + logical_positions, + compressed_slots, + k_work_metadata, + compressed_cache, + compressed_cache.stride(0), + compressed_cache.stride(1), + num_tokens, + state_cache.shape[0], + compressed_cache.shape[0], + num_k_work, + HQ=num_q_heads, + D=head_dim, + TILE_T_Q=TILE_T_Q, + TILE_H_Q=TILE_H_Q, + COMPRESS_RATIO=compress_ratio, + STATE_SIZE=state_cache.shape[1], + COMP_PAGE_SIZE=compressed_cache.shape[1], + IS_2D_POSITIONS=is_2d_positions, + IS_K_MROPE=is_k_mrope, + CACHE_HAS_ROPE_POS=cache_has_rope_pos, + MROPE_H=section[1], + MROPE_W=section[2], + num_warps=1, + ) + + +__all__ = ["qsa_pre_indexer", "supports_fused_pre_indexer"]