[ROCm][Perf] Reach the fused QSA pre-indexer from the AMD path (#57947)
Signed-off-by: mjkvaak-amd <mjkvaak-amd@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
528772a4bf
commit
d23af12cb1
@@ -206,7 +206,7 @@ steps:
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
commands:
|
||||
- pytest -v -s models/qwen4_exp/test_qsa_amd.py
|
||||
- pytest -v -s models/qwen4_exp/test_qsa_amd.py models/qwen4_exp/test_qsa_pre_indexer_amd.py
|
||||
# PLE and QSA import conflicting HC registrations in a shared process.
|
||||
- pytest -v -s models/qwen4_exp/test_ple.py
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import inspect
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
@@ -9,15 +10,21 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.models.qwen4_exp.amd import indexer_qsa
|
||||
from vllm.models.qwen4_exp.amd import (
|
||||
model as _qwen4_exp_model, # noqa: F401
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd import ple_layer as ple_layer_module
|
||||
from vllm.models.qwen4_exp.amd.indexer_qsa import (
|
||||
QSAIndexer,
|
||||
apply_qsa_rmsnorm,
|
||||
apply_qsa_rope,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import (
|
||||
qsa_pre_indexer,
|
||||
supports_fused_pre_indexer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
@@ -120,6 +127,149 @@ def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
|
||||
torch.testing.assert_close(output, norm.forward_native(tensor))
|
||||
|
||||
|
||||
def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer:
|
||||
"""A QSAIndexer whose unfused path runs for real, with no projection weights.
|
||||
|
||||
Both branches have to be reachable: a fused-path test is only meaningful if
|
||||
the unfused one it replaces would otherwise have executed.
|
||||
"""
|
||||
heads, head_dim = 2, 4
|
||||
indexer = QSAIndexer.__new__(QSAIndexer)
|
||||
torch.nn.Module.__init__(indexer)
|
||||
indexer.index_n_heads = heads
|
||||
indexer.index_kv_heads = 1
|
||||
indexer.index_head_dim = head_dim
|
||||
indexer.compress_ratio = 2
|
||||
indexer.use_fused_pre_indexer = use_fused
|
||||
indexer.skip_topk = True
|
||||
indexer.index_qk_proj = lambda hidden_states: (
|
||||
torch.zeros(hidden_states.shape[0], (heads + 1) * head_dim),
|
||||
None,
|
||||
)
|
||||
indexer.q_layernorm = GemmaRMSNorm(head_dim, eps=1e-6)
|
||||
indexer.k_layernorm = GemmaRMSNorm(head_dim, eps=1e-6)
|
||||
indexer.rotary_emb = SimpleNamespace(
|
||||
cos_sin_cache=torch.zeros(16, head_dim),
|
||||
rotary_dim=head_dim // 2,
|
||||
apply_rotary_emb=lambda tensor, cos, sin: tensor,
|
||||
_match_cos_sin_cache_dtype=lambda _: torch.zeros(16, head_dim),
|
||||
)
|
||||
# The raw cache's full width carries the MRoPE position tail that the fused
|
||||
# kernel writes; key_cache is only the leading key columns.
|
||||
indexer.raw_key_cache = SimpleNamespace(
|
||||
kv_cache=torch.zeros(1, 4, 1, head_dim + 12),
|
||||
key_cache=torch.zeros(1, 4, 1, head_dim),
|
||||
cache_rope_positions=True,
|
||||
rope_position_offset=head_dim,
|
||||
)
|
||||
indexer.compressed_key_cache = SimpleNamespace(
|
||||
kv_cache=torch.zeros(1, 2, 1, head_dim)
|
||||
)
|
||||
raw_metadata = SimpleNamespace(
|
||||
num_actual_tokens=num_tokens,
|
||||
slot_mapping=torch.arange(num_tokens),
|
||||
block_table=torch.zeros(1, 1, dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, num_tokens], dtype=torch.int32),
|
||||
logical_positions=torch.arange(num_tokens),
|
||||
token_to_req=torch.zeros(num_tokens, dtype=torch.int32),
|
||||
)
|
||||
compressed_metadata = SimpleNamespace(
|
||||
num_actual_tokens=num_tokens,
|
||||
slot_mapping=torch.arange(num_tokens),
|
||||
k_work_metadata=torch.zeros(1, 2, dtype=torch.int32),
|
||||
)
|
||||
indexer._metadata = lambda: (raw_metadata, compressed_metadata)
|
||||
indexer.test_metadata = (raw_metadata, compressed_metadata)
|
||||
return indexer
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_qsa_fused_pre_indexer_replaces_the_unfused_chain(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
indexer = _make_indexer(use_fused=True)
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
QSAIndexer,
|
||||
"_update_and_compress",
|
||||
lambda *args, **kwargs: pytest.fail("fused path ran the unfused chain"),
|
||||
)
|
||||
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
|
||||
|
||||
actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows)
|
||||
|
||||
assert actual is rows
|
||||
assert len(calls) == 1
|
||||
args, kwargs = calls[0]
|
||||
passed = inspect.signature(qsa_pre_indexer).bind(*args, **kwargs).arguments
|
||||
_, compressed_metadata = indexer.test_metadata
|
||||
# The kernel addresses the raw cache at its full width and writes the MRoPE
|
||||
# position tail itself, so it needs kv_cache, not the key_cache view.
|
||||
assert passed["state_cache"] is indexer.raw_key_cache.kv_cache
|
||||
assert passed["compressed_cache"] is indexer.compressed_key_cache.kv_cache
|
||||
assert passed["k_work_metadata"] is compressed_metadata.k_work_metadata
|
||||
assert passed["compress_ratio"] == indexer.compress_ratio
|
||||
assert passed["rope_pos_offset"] == indexer.index_head_dim
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_qsa_unsupported_config_keeps_the_unfused_chain(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
indexer = _make_indexer(use_fused=False)
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
lambda *args, **kwargs: pytest.fail("unfused path ran the fused kernel"),
|
||||
)
|
||||
updates = []
|
||||
monkeypatch.setattr(
|
||||
QSAIndexer,
|
||||
"_update_and_compress",
|
||||
lambda *args, **kwargs: updates.append(args),
|
||||
)
|
||||
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
|
||||
|
||||
actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows)
|
||||
|
||||
assert actual is rows
|
||||
assert len(updates) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("overrides", "expected"),
|
||||
[
|
||||
pytest.param({}, True, id="qwen3_8_flash_next"),
|
||||
pytest.param({"is_neox_style": False}, False, id="not-neox"),
|
||||
pytest.param({"rotary_dim": 128}, False, id="rotary-dim"),
|
||||
pytest.param({"mrope_interleaved": False}, False, id="mrope-not-interleaved"),
|
||||
pytest.param({"mrope_section": (11, 11, 10, 0)}, False, id="mrope-section-len"),
|
||||
],
|
||||
)
|
||||
def test_fused_pre_indexer_gate_is_config_shaped(overrides, expected) -> None:
|
||||
# Qwen3.8-Flash-Next's indexer geometry, which both vendors satisfy.
|
||||
rotary = SimpleNamespace(
|
||||
rotary_dim=64,
|
||||
is_neox_style=True,
|
||||
mrope_section=(11, 11, 10),
|
||||
mrope_interleaved=True,
|
||||
)
|
||||
for name, value in overrides.items():
|
||||
setattr(rotary, name, value)
|
||||
|
||||
assert (
|
||||
supports_fused_pre_indexer(
|
||||
rotary, head_dim=128, num_kv_heads=1, compress_ratio=4
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
def test_qsa_selection_uses_portable_topk_on_rocm(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the AMD fused QSA pre-indexer."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.models.qwen4_exp.amd.indexer_qsa import apply_qsa_rope
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa import (
|
||||
qsa_compress_groups_with_ratio,
|
||||
qsa_store_cache_rows,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import (
|
||||
qsa_pre_indexer,
|
||||
)
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import (
|
||||
canonical_qsa_rope_positions,
|
||||
circular_qsa_slot_mapping,
|
||||
compressed_qsa_slot_mapping,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not HAS_TRITON,
|
||||
reason="the AMD QSA kernels require ROCm and Triton",
|
||||
)
|
||||
|
||||
|
||||
def _reference_gemma_rmsnorm(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""The unfused Gemma RMSNorm the AMD reference path actually runs."""
|
||||
norm = GemmaRMSNorm(weight.shape[-1], eps=eps).to(
|
||||
device=weight.device, dtype=weight.dtype
|
||||
)
|
||||
with torch.no_grad():
|
||||
norm.weight.copy_(weight)
|
||||
return norm.forward_native(x)
|
||||
|
||||
|
||||
HQ, D = 4, 128
|
||||
CR = 4
|
||||
MROPE_SECTION = (11, 11, 10)
|
||||
EPS = 1e-6
|
||||
BLOCK_SIZE = 16
|
||||
COMP_PAGE = BLOCK_SIZE // CR
|
||||
ROPE_POS_OFFSET = D
|
||||
RTOL = 1.6e-2
|
||||
ATOL = 1e-2
|
||||
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
|
||||
|
||||
|
||||
def _make_block_table(block_counts):
|
||||
num_blocks = sum(block_counts)
|
||||
table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32)
|
||||
physical_blocks = torch.randperm(num_blocks)
|
||||
offset = 0
|
||||
for request, count in enumerate(block_counts):
|
||||
table[request, :count] = physical_blocks[offset : offset + count]
|
||||
offset += count
|
||||
return table, num_blocks
|
||||
|
||||
|
||||
def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
# e4m3 is sign-magnitude, so within a sign the uint8 code order matches the
|
||||
# value order and one ulp is one code step. The two paths' intermediates
|
||||
# differ in the pooling accumulation order, which at denormal magnitudes
|
||||
# (absolute grid step 2^-9) shows up as up to 2 code steps.
|
||||
code_diff = (
|
||||
actual.view(torch.uint8).int() - expected.view(torch.uint8).int()
|
||||
).abs()
|
||||
abs_diff = (actual.float() - expected.float()).abs()
|
||||
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize(
|
||||
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
|
||||
[
|
||||
pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"),
|
||||
pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"),
|
||||
pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"),
|
||||
pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"),
|
||||
pytest.param(
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
8,
|
||||
*MIXED_BATCH,
|
||||
id="mrope-model-1d-no-position-cache",
|
||||
),
|
||||
pytest.param(
|
||||
False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache"
|
||||
),
|
||||
pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"),
|
||||
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
|
||||
],
|
||||
)
|
||||
def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
indexer_dtype,
|
||||
mrope,
|
||||
is_2d_positions,
|
||||
cache_rope_positions,
|
||||
state_size,
|
||||
seq_lens,
|
||||
query_lens,
|
||||
history_lens,
|
||||
) -> None:
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
|
||||
if indexer_dtype == torch.float8_e4m3fn:
|
||||
pytest.skip("the AMD QSA indexer cache is bf16-only")
|
||||
|
||||
# The FP8 comparison below is a one-ulp bound on data this test generates,
|
||||
# so an unseeded run decides for itself whether it reproduces a rounding
|
||||
# difference. Seed it, or the kernel's bit-exactness is only sampled.
|
||||
torch.manual_seed(0)
|
||||
|
||||
device = "cuda"
|
||||
rope_params = {
|
||||
"partial_rotary_factor": 0.25,
|
||||
"rope_theta": 10000000,
|
||||
"rope_type": "default",
|
||||
}
|
||||
if mrope:
|
||||
rope_params["mrope_interleaved"] = True
|
||||
rope_params["mrope_section"] = list(MROPE_SECTION)
|
||||
with torch.device(device):
|
||||
rope = get_rope(
|
||||
head_size=256,
|
||||
max_position=32768,
|
||||
rope_parameters=rope_params,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
token_to_req = torch.cat(
|
||||
[
|
||||
torch.full((length,), request, dtype=torch.int32)
|
||||
for request, length in enumerate(query_lens)
|
||||
]
|
||||
).to(device)
|
||||
logical_positions = torch.cat(
|
||||
[
|
||||
torch.arange(seq_len - query_len, seq_len, dtype=torch.int64)
|
||||
for seq_len, query_len in zip(seq_lens, query_lens)
|
||||
]
|
||||
).to(device)
|
||||
num_tokens = logical_positions.numel()
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
|
||||
).to(device)
|
||||
positions = (
|
||||
torch.stack(
|
||||
[
|
||||
logical_positions,
|
||||
logical_positions // 7 + 3,
|
||||
logical_positions // 13 + 11,
|
||||
]
|
||||
)
|
||||
if is_2d_positions
|
||||
else logical_positions
|
||||
)
|
||||
position_rows = (
|
||||
canonical_qsa_rope_positions(positions)
|
||||
if cache_rope_positions
|
||||
else logical_positions.view(-1, 1, 1).expand(-1, 1, 3)
|
||||
)
|
||||
|
||||
compressed_block_counts = [
|
||||
(seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens
|
||||
]
|
||||
raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens))
|
||||
compressed_block_table, num_compressed_blocks = _make_block_table(
|
||||
compressed_block_counts
|
||||
)
|
||||
raw_block_table = raw_block_table.to(device)
|
||||
raw_slots = circular_qsa_slot_mapping(
|
||||
raw_block_table,
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
state_size,
|
||||
query_start_loc,
|
||||
)
|
||||
compressed_slots = compressed_qsa_slot_mapping(
|
||||
compressed_block_table.to(device),
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
COMP_PAGE,
|
||||
CR,
|
||||
)
|
||||
group_counts = torch.tensor(
|
||||
[
|
||||
seq_len // CR - (seq_len - query_len) // CR
|
||||
for seq_len, query_len in zip(seq_lens, query_lens)
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts))
|
||||
k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)])
|
||||
work_requests = torch.repeat_interleave(
|
||||
torch.arange(len(query_lens), dtype=torch.int32, device=device),
|
||||
k_work_counts,
|
||||
)
|
||||
local_work = torch.arange(
|
||||
int(k_start_loc[-1]), dtype=torch.int32, device=device
|
||||
) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts)
|
||||
max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR
|
||||
k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device)
|
||||
k_work_metadata[: work_requests.numel()] = torch.stack(
|
||||
(work_requests, local_work), dim=1
|
||||
)
|
||||
|
||||
raw_width = D + 12 if cache_rope_positions else D
|
||||
# Match vLLM's padded-page cache layout: rows are contiguous, while physical
|
||||
# blocks have a larger stride than their logical contents.
|
||||
raw_page_elements = state_size * raw_width
|
||||
fused_raw_storage = torch.zeros(
|
||||
num_raw_blocks,
|
||||
raw_page_elements + 16,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
fused_raw = torch.as_strided(
|
||||
fused_raw_storage,
|
||||
(num_raw_blocks, state_size, 1, raw_width),
|
||||
(raw_page_elements + 16, raw_width, raw_width, 1),
|
||||
)
|
||||
for request, history_len in enumerate(history_lens):
|
||||
history_end = seq_lens[request] - query_lens[request]
|
||||
for position in range(history_end - history_len, history_end):
|
||||
block = int(raw_block_table[request, 0])
|
||||
row = fused_raw[block, position % state_size, 0]
|
||||
row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device)
|
||||
if cache_rope_positions:
|
||||
row[ROPE_POS_OFFSET:].view(torch.int64).copy_(
|
||||
torch.tensor(
|
||||
[position, position // 7 + 3, position // 13 + 11],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
unfused_raw = fused_raw.clone()
|
||||
compressed_page_elements = COMP_PAGE * D
|
||||
fused_compressed_storage = torch.zeros(
|
||||
num_compressed_blocks,
|
||||
compressed_page_elements + 16,
|
||||
dtype=indexer_dtype,
|
||||
device=device,
|
||||
)
|
||||
fused_compressed = torch.as_strided(
|
||||
fused_compressed_storage,
|
||||
(num_compressed_blocks, COMP_PAGE, 1, D),
|
||||
(compressed_page_elements + 16, D, D, 1),
|
||||
)
|
||||
unfused_compressed = fused_compressed.clone()
|
||||
|
||||
projected_qk = torch.randn(
|
||||
num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
|
||||
qsa_pre_indexer(
|
||||
projected_qk[:, : HQ * D],
|
||||
projected_qk[:, HQ * D :],
|
||||
positions,
|
||||
rope.cos_sin_cache,
|
||||
q_weight,
|
||||
k_weight,
|
||||
EPS,
|
||||
fused_query,
|
||||
fused_raw,
|
||||
raw_slots,
|
||||
raw_block_table,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
fused_compressed,
|
||||
compressed_slots,
|
||||
k_work_metadata,
|
||||
compress_ratio=CR,
|
||||
mrope_section=MROPE_SECTION if mrope else None,
|
||||
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
|
||||
)
|
||||
|
||||
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
|
||||
unfused_query = _reference_gemma_rmsnorm(
|
||||
unfused_query.reshape(-1, D), q_weight, EPS
|
||||
).reshape_as(unfused_query)
|
||||
unfused_query = apply_qsa_rope(rope, positions, unfused_query)
|
||||
|
||||
raw_keys = unfused_raw[..., :D]
|
||||
rope_positions = (
|
||||
unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64)
|
||||
if cache_rope_positions
|
||||
else None
|
||||
)
|
||||
pooled, first_positions = qsa_compress_groups_with_ratio(
|
||||
projected_qk[:, HQ * D :].reshape(-1, 1, D),
|
||||
position_rows,
|
||||
raw_keys,
|
||||
raw_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
CR,
|
||||
rope_positions,
|
||||
)
|
||||
compressed_rows = _reference_gemma_rmsnorm(
|
||||
pooled.reshape(-1, D), k_weight, EPS
|
||||
).reshape(-1, 1, D)
|
||||
group_positions = (
|
||||
first_positions.transpose(0, 1) if mrope else first_positions[:, 0]
|
||||
)
|
||||
compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows)
|
||||
qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows)
|
||||
qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :])
|
||||
if rope_positions is not None:
|
||||
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
|
||||
|
||||
if indexer_dtype == torch.float8_e4m3fn:
|
||||
# Both paths round the same ~bf16 intermediates to e4m3. Bitwise
|
||||
# equality (the dsv4 indexer test's bar) does not hold here: the 1D
|
||||
# reference rope (forward_cuda) differs from _norm_rope by up to 1
|
||||
# bf16 ulp, and even the MRoPE pair flips a code occasionally.
|
||||
unfused_query = unfused_query.to(indexer_dtype)
|
||||
assert_fp8_within_one_ulp(fused_query, unfused_query)
|
||||
else:
|
||||
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
|
||||
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
|
||||
if indexer_dtype == torch.float8_e4m3fn:
|
||||
assert_fp8_within_one_ulp(fused_compressed, unfused_compressed)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
|
||||
)
|
||||
@@ -23,6 +23,7 @@ from ..common.qsa_cache import (
|
||||
QSAKeyStateCache,
|
||||
canonical_qsa_rope_positions,
|
||||
)
|
||||
from .ops.qsa_pre_indexer import qsa_pre_indexer, supports_fused_pre_indexer
|
||||
|
||||
|
||||
def apply_qsa_rope(
|
||||
@@ -98,6 +99,12 @@ class QSAIndexer(nn.Module):
|
||||
self.token_topk = int(config.indexer_budget)
|
||||
self.compress_ratio = int(config.indexer_compress_ratio)
|
||||
self.rotary_emb = rotary_emb
|
||||
self.use_fused_pre_indexer = supports_fused_pre_indexer(
|
||||
rotary_emb,
|
||||
self.index_head_dim,
|
||||
self.index_kv_heads,
|
||||
self.compress_ratio,
|
||||
)
|
||||
self.prefix = prefix
|
||||
# MTP step 0 selects the target-aligned rows; later steps reuse them
|
||||
# while continuing to update the QSA side cache.
|
||||
@@ -143,20 +150,27 @@ class QSAIndexer(nn.Module):
|
||||
def output_width(self) -> int:
|
||||
return self.token_topk + self.compress_ratio - 1
|
||||
|
||||
def project_qk(
|
||||
def project(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Project replicated Q/K, normalize+rotate Q, and preserve raw K."""
|
||||
"""Project replicated Q/K and split, leaving both operands unrotated."""
|
||||
qk, _ = self.index_qk_proj(hidden_states)
|
||||
q_raw, token_k = qk.split(
|
||||
return qk.split(
|
||||
(
|
||||
self.index_n_heads * self.index_head_dim,
|
||||
self.index_kv_heads * self.index_head_dim,
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
def project_qk(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Project replicated Q/K, normalize+rotate Q, and preserve raw K."""
|
||||
q_raw, token_k = self.project(hidden_states)
|
||||
q = q_raw.reshape(-1, self.index_n_heads, self.index_head_dim)
|
||||
q = apply_qsa_rmsnorm(
|
||||
self.q_layernorm,
|
||||
@@ -251,6 +265,52 @@ class QSAIndexer(nn.Module):
|
||||
position_rows,
|
||||
)
|
||||
|
||||
def _fused_pre_indexer(
|
||||
self,
|
||||
projected_q: torch.Tensor,
|
||||
raw_keys: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
raw_metadata: QSAForwardMetadata,
|
||||
compressed_metadata: QSAForwardMetadata,
|
||||
) -> torch.Tensor:
|
||||
"""Normalize and rotate Q, compress K, and write both caches in one launch.
|
||||
|
||||
Replaces ``project_qk``'s norm/RoPE half and the whole of
|
||||
``_update_and_compress``; the two must stay numerically interchangeable.
|
||||
"""
|
||||
raw_key_cache = self.raw_key_cache
|
||||
q = projected_q.new_empty(
|
||||
raw_metadata.num_actual_tokens,
|
||||
self.index_n_heads,
|
||||
self.index_head_dim,
|
||||
)
|
||||
qsa_pre_indexer(
|
||||
projected_q,
|
||||
raw_keys,
|
||||
positions,
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.q_layernorm.weight,
|
||||
self.k_layernorm.weight,
|
||||
self.q_layernorm.variance_epsilon,
|
||||
q,
|
||||
raw_key_cache.kv_cache,
|
||||
raw_metadata.slot_mapping,
|
||||
raw_metadata.block_table,
|
||||
raw_metadata.query_start_loc,
|
||||
raw_metadata.logical_positions,
|
||||
self.compressed_key_cache.kv_cache,
|
||||
compressed_metadata.slot_mapping,
|
||||
compressed_metadata.k_work_metadata,
|
||||
compress_ratio=self.compress_ratio,
|
||||
mrope_section=getattr(self.rotary_emb, "mrope_section", None),
|
||||
rope_pos_offset=(
|
||||
raw_key_cache.rope_position_offset
|
||||
if raw_key_cache.cache_rope_positions
|
||||
else None
|
||||
),
|
||||
)
|
||||
return q
|
||||
|
||||
def _select(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -295,15 +355,25 @@ class QSAIndexer(nn.Module):
|
||||
return result
|
||||
raw_metadata, compressed_metadata = metadata
|
||||
num_tokens = raw_metadata.num_actual_tokens
|
||||
q, token_k = self.project_qk(
|
||||
hidden_states[:num_tokens], positions[..., :num_tokens]
|
||||
)
|
||||
self._update_and_compress(
|
||||
token_k,
|
||||
positions[..., :num_tokens],
|
||||
raw_metadata,
|
||||
compressed_metadata,
|
||||
)
|
||||
hidden_states = hidden_states[:num_tokens]
|
||||
positions = positions[..., :num_tokens]
|
||||
if self.use_fused_pre_indexer:
|
||||
projected_q, raw_keys = self.project(hidden_states)
|
||||
q = self._fused_pre_indexer(
|
||||
projected_q,
|
||||
raw_keys,
|
||||
positions,
|
||||
raw_metadata,
|
||||
compressed_metadata,
|
||||
)
|
||||
else:
|
||||
q, token_k = self.project_qk(hidden_states, positions)
|
||||
self._update_and_compress(
|
||||
token_k,
|
||||
positions,
|
||||
raw_metadata,
|
||||
compressed_metadata,
|
||||
)
|
||||
if self.skip_topk:
|
||||
if out is None:
|
||||
raise RuntimeError("QSA top-k reuse requires an output buffer")
|
||||
|
||||
@@ -0,0 +1,558 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused QSA pre-indexer kernel for the AMD Qwen4Exp path.
|
||||
|
||||
Started as a copy of the NVIDIA kernel (``nvidia/ops/qsa_prepare.py``) and is
|
||||
kept separate so either vendor can change its own copy without re-validating
|
||||
the other. The state source select is two masked loads rather than a pointer
|
||||
select, which ROCm Triton rejects.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def supports_fused_pre_indexer(
|
||||
rotary_emb: nn.Module,
|
||||
head_dim: int,
|
||||
num_kv_heads: int,
|
||||
compress_ratio: int,
|
||||
) -> bool:
|
||||
"""Report whether this indexer's shapes match the fused kernel's assumptions.
|
||||
|
||||
The kernel hard-codes the rotary layout and the single-KV-head group
|
||||
compression it was written for; everything it rejects has a working unfused
|
||||
path.
|
||||
"""
|
||||
rotary_dim = int(rotary_emb.rotary_dim)
|
||||
mrope_section = getattr(rotary_emb, "mrope_section", None)
|
||||
return (
|
||||
bool(getattr(rotary_emb, "is_neox_style", False))
|
||||
and (
|
||||
not mrope_section
|
||||
or (
|
||||
len(mrope_section) == 3
|
||||
and sum(mrope_section) == rotary_dim // 2
|
||||
and bool(getattr(rotary_emb, "mrope_interleaved", False))
|
||||
)
|
||||
)
|
||||
and head_dim == 128
|
||||
and rotary_dim == 64
|
||||
and num_kv_heads == 1
|
||||
and compress_ratio > 1
|
||||
and compress_ratio & (compress_ratio - 1) == 0
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _norm_rope(
|
||||
x,
|
||||
pos_t,
|
||||
pos_h,
|
||||
pos_w,
|
||||
cos_sin_ptr,
|
||||
cos_sin_stride,
|
||||
norm_weight_ptr,
|
||||
eps,
|
||||
IS_MROPE: tl.constexpr,
|
||||
MROPE_H: tl.constexpr,
|
||||
MROPE_W: tl.constexpr,
|
||||
):
|
||||
"""Apply Gemma RMSNorm and selected-axis NeoX RoPE to register rows."""
|
||||
TILE_T: tl.constexpr = x.shape[0]
|
||||
TILE_H: tl.constexpr = x.shape[1]
|
||||
D: tl.constexpr = x.shape[2]
|
||||
ROWS: tl.constexpr = TILE_T * TILE_H
|
||||
HALF: tl.constexpr = D // 2
|
||||
QUARTER: tl.constexpr = D // 4
|
||||
pairs = tl.arange(0, QUARTER)
|
||||
if IS_MROPE:
|
||||
# Qwen interleaves temporal, height, and width rotary pairs. Each axis
|
||||
# still indexes the same position-major cos/sin table.
|
||||
h_mask = ((pairs % 3) == 1) & (pairs <= 3 * MROPE_H)
|
||||
w_mask = ((pairs % 3) == 2) & (pairs <= 3 * MROPE_W)
|
||||
t_mask = ~(h_mask | w_mask)
|
||||
base = cos_sin_ptr + pairs[None, :]
|
||||
pos_rows = (pos_t, pos_h, pos_w)
|
||||
axis_masks = (t_mask, h_mask, w_mask)
|
||||
cos = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty)
|
||||
sin = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty)
|
||||
for axis in tl.static_range(3):
|
||||
cos += tl.load(
|
||||
base + pos_rows[axis][:, None] * cos_sin_stride,
|
||||
mask=axis_masks[axis][None, :],
|
||||
other=0,
|
||||
)
|
||||
sin += tl.load(
|
||||
base + pos_rows[axis][:, None] * cos_sin_stride + QUARTER,
|
||||
mask=axis_masks[axis][None, :],
|
||||
other=0,
|
||||
)
|
||||
else:
|
||||
cos = tl.load(cos_sin_ptr + pos_t[:, None] * cos_sin_stride + pairs[None, :])
|
||||
sin = tl.load(
|
||||
cos_sin_ptr + pos_t[:, None] * cos_sin_stride + QUARTER + pairs[None, :]
|
||||
)
|
||||
|
||||
cos = tl.reshape(
|
||||
tl.broadcast_to(cos[:, None, :], (TILE_T, TILE_H, QUARTER)),
|
||||
(ROWS, QUARTER),
|
||||
)
|
||||
sin = tl.reshape(
|
||||
tl.broadcast_to(sin[:, None, :], (TILE_T, TILE_H, QUARTER)),
|
||||
(ROWS, QUARTER),
|
||||
)
|
||||
x = tl.reshape(x, (ROWS, D)).to(tl.float32)
|
||||
weight = tl.load(norm_weight_ptr + tl.arange(0, D)).to(tl.float32) + 1.0
|
||||
rrms = tl.rsqrt(tl.sum(x * x, axis=1) / D + eps)
|
||||
y = (x * rrms[:, None] * weight[None, :]).to(cos.dtype)
|
||||
rotated, passthrough = tl.split(
|
||||
tl.permute(tl.reshape(y, (ROWS, 2, HALF)), (0, 2, 1))
|
||||
)
|
||||
r0, r1 = tl.split(tl.permute(tl.reshape(rotated, (ROWS, 2, QUARTER)), (0, 2, 1)))
|
||||
out0 = r0 * cos - r1 * sin
|
||||
out1 = r1 * cos + r0 * sin
|
||||
rotated = tl.reshape(tl.permute(tl.join(out0, out1), (0, 2, 1)), (ROWS, HALF))
|
||||
result = tl.reshape(tl.permute(tl.join(rotated, passthrough), (0, 2, 1)), (ROWS, D))
|
||||
return tl.reshape(result, (TILE_T, TILE_H, D))
|
||||
|
||||
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"num_tokens",
|
||||
"num_state_blocks",
|
||||
"num_compressed_blocks",
|
||||
"num_k_work",
|
||||
]
|
||||
)
|
||||
def _qsa_pre_indexer_kernel(
|
||||
q_ptr,
|
||||
q_stride_token,
|
||||
k_ptr,
|
||||
k_stride_token,
|
||||
pos_ptr,
|
||||
pos_stride_axis,
|
||||
pos_stride_token,
|
||||
cos_sin_ptr,
|
||||
q_norm_weight_ptr,
|
||||
k_norm_weight_ptr,
|
||||
eps,
|
||||
q_out_ptr,
|
||||
q_out_stride_token,
|
||||
q_out_stride_head,
|
||||
state_cache_ptr,
|
||||
state_cache_stride_block,
|
||||
state_cache_stride_token,
|
||||
state_slots_ptr,
|
||||
state_table_ptr,
|
||||
state_table_stride_req,
|
||||
query_start_loc_ptr,
|
||||
logical_positions_ptr,
|
||||
compressed_slots_ptr,
|
||||
k_work_metadata_ptr,
|
||||
compressed_cache_ptr,
|
||||
compressed_cache_stride_block,
|
||||
compressed_cache_stride_token,
|
||||
num_tokens,
|
||||
num_state_blocks,
|
||||
num_compressed_blocks,
|
||||
num_k_work,
|
||||
HQ: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
TILE_T_Q: tl.constexpr,
|
||||
TILE_H_Q: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
STATE_SIZE: tl.constexpr,
|
||||
COMP_PAGE_SIZE: tl.constexpr,
|
||||
IS_2D_POSITIONS: tl.constexpr,
|
||||
IS_K_MROPE: tl.constexpr,
|
||||
CACHE_HAS_ROPE_POS: tl.constexpr,
|
||||
MROPE_H: tl.constexpr,
|
||||
MROPE_W: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
# K work occupies the first programs; the remaining programs tile Q. This
|
||||
# keeps both paths in one launch while leaving their register shapes
|
||||
# independent.
|
||||
if pid >= num_k_work:
|
||||
q_pid = pid - num_k_work
|
||||
num_head_tiles: tl.constexpr = tl.cdiv(HQ, TILE_H_Q)
|
||||
token_tile = q_pid // num_head_tiles
|
||||
head_tile = q_pid % num_head_tiles
|
||||
tokens = token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q)
|
||||
heads = head_tile * TILE_H_Q + tl.arange(0, TILE_H_Q)
|
||||
valid_tokens = tokens < num_tokens
|
||||
valid_heads = heads < HQ
|
||||
dims = tl.arange(0, D)
|
||||
mask = valid_tokens[:, None, None] & valid_heads[None, :, None]
|
||||
x = tl.load(
|
||||
q_ptr
|
||||
+ tokens[:, None, None] * q_stride_token
|
||||
+ heads[None, :, None] * D
|
||||
+ dims[None, None, :],
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
pos_t = tl.load(pos_ptr + tokens * pos_stride_token, mask=valid_tokens, other=0)
|
||||
if IS_2D_POSITIONS:
|
||||
pos_h = tl.load(
|
||||
pos_ptr + pos_stride_axis + tokens * pos_stride_token,
|
||||
mask=valid_tokens,
|
||||
other=0,
|
||||
)
|
||||
pos_w = tl.load(
|
||||
pos_ptr + 2 * pos_stride_axis + tokens * pos_stride_token,
|
||||
mask=valid_tokens,
|
||||
other=0,
|
||||
)
|
||||
else:
|
||||
pos_h = pos_t
|
||||
pos_w = pos_t
|
||||
y = _norm_rope(
|
||||
x,
|
||||
pos_t,
|
||||
pos_h,
|
||||
pos_w,
|
||||
cos_sin_ptr,
|
||||
D // 2,
|
||||
q_norm_weight_ptr,
|
||||
eps,
|
||||
IS_2D_POSITIONS,
|
||||
MROPE_H,
|
||||
MROPE_W,
|
||||
)
|
||||
tl.store(
|
||||
q_out_ptr
|
||||
+ tokens[:, None, None] * q_out_stride_token
|
||||
+ heads[None, :, None] * q_out_stride_head
|
||||
+ dims[None, None, :],
|
||||
y,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
if pid < num_k_work:
|
||||
# One work item owns one completed compression group. Work item zero
|
||||
# additionally commits the request's current raw-K suffix below.
|
||||
work_metadata = tl.load(k_work_metadata_ptr + pid * 2 + tl.arange(0, 2))
|
||||
request, work_in_request = tl.split(work_metadata)
|
||||
if request < 0:
|
||||
return
|
||||
|
||||
query_start = tl.load(query_start_loc_ptr + request)
|
||||
query_end = tl.load(query_start_loc_ptr + request + 1)
|
||||
query_len = query_end - query_start
|
||||
chunk_end = tl.load(logical_positions_ptr + query_end - 1)
|
||||
chunk_start = chunk_end - query_len + 1
|
||||
num_groups = (chunk_end + 1) // COMPRESS_RATIO - chunk_start // COMPRESS_RATIO
|
||||
dims = tl.arange(0, D)
|
||||
|
||||
if work_in_request < num_groups:
|
||||
first_boundary = (
|
||||
(chunk_start + COMPRESS_RATIO) // COMPRESS_RATIO
|
||||
) * COMPRESS_RATIO - 1
|
||||
end_position = first_boundary + work_in_request * COMPRESS_RATIO
|
||||
boundary_token = query_start + end_position - chunk_start
|
||||
valid_token = (
|
||||
(boundary_token >= query_start)
|
||||
& (boundary_token < query_end)
|
||||
& (boundary_token < num_tokens)
|
||||
)
|
||||
compressed_slot = tl.load(
|
||||
compressed_slots_ptr + boundary_token,
|
||||
mask=valid_token,
|
||||
other=-1,
|
||||
)
|
||||
valid = (
|
||||
valid_token
|
||||
& (compressed_slot >= 0)
|
||||
& (compressed_slot < num_compressed_blocks * COMP_PAGE_SIZE)
|
||||
)
|
||||
state_block = tl.load(state_table_ptr + request * state_table_stride_req)
|
||||
state_block_valid = (state_block >= 0) & (state_block < num_state_blocks)
|
||||
safe_state_block = tl.maximum(state_block, 0).to(tl.int64)
|
||||
group_offsets = tl.arange(0, COMPRESS_RATIO)
|
||||
source_positions = end_position - (COMPRESS_RATIO - 1) + group_offsets
|
||||
source_in_chunk = source_positions >= chunk_start
|
||||
source_in_cache = source_positions < chunk_start
|
||||
source_tokens = query_start + source_positions - chunk_start
|
||||
source_tokens_valid = (
|
||||
(source_tokens >= query_start)
|
||||
& (source_tokens < query_end)
|
||||
& (source_tokens < num_tokens)
|
||||
)
|
||||
current_base = (
|
||||
k_ptr + tl.maximum(source_tokens, 0)[:, None] * k_stride_token
|
||||
)
|
||||
cached_base = (
|
||||
state_cache_ptr
|
||||
+ safe_state_block * state_cache_stride_block
|
||||
+ (source_positions % STATE_SIZE)[:, None] * state_cache_stride_token
|
||||
)
|
||||
# Only the first completed group can cross the chunk boundary, so
|
||||
# every row comes from exactly one of the two sources. Selecting
|
||||
# between the base pointers would need a pointer-typed tl.where,
|
||||
# which ROCm's Triton rejects ("'tt.addptr' op failed to verify
|
||||
# that result type matches ptr type"), so select the loaded values
|
||||
# instead. The two masks are complementary, so no lane fetches from
|
||||
# both and memory traffic is unchanged.
|
||||
#
|
||||
# Both bases still need the alignment stated explicitly. Without it
|
||||
# Triton vectorizes these loads differently, which reorders the
|
||||
# pooling reduction below; float addition is not associative, so the
|
||||
# compressed rows stop matching the unfused path bit for bit and an
|
||||
# FP8 cache rounds a near-zero pooled value to the wrong code.
|
||||
current_base = tl.multiple_of(current_base, (8, 8))
|
||||
cached_base = tl.multiple_of(cached_base, (8, 8))
|
||||
from_chunk = valid & source_in_chunk[:, None] & source_tokens_valid[:, None]
|
||||
from_cache = valid & source_in_cache[:, None] & state_block_valid
|
||||
current = tl.load(current_base + dims[None, :], mask=from_chunk, other=0.0)
|
||||
cached = tl.load(cached_base + dims[None, :], mask=from_cache, other=0.0)
|
||||
source = tl.where(source_in_chunk[:, None], current, cached).to(tl.float32)
|
||||
# Match the unfused path's BF16 pooled tensor before RMSNorm.
|
||||
pooled = (
|
||||
(tl.sum(source, axis=0) / COMPRESS_RATIO).to(tl.bfloat16).to(tl.float32)
|
||||
)
|
||||
|
||||
first_position = end_position - (COMPRESS_RATIO - 1)
|
||||
if CACHE_HAS_ROPE_POS:
|
||||
# RoPE uses the first token in the pooled group. Its exact MRoPE
|
||||
# coordinates may live in this chunk or the raw-state ring.
|
||||
first_in_chunk = first_position >= chunk_start
|
||||
first_token = query_start + first_position - chunk_start
|
||||
first_token_valid = (
|
||||
(first_token >= query_start)
|
||||
& (first_token < query_end)
|
||||
& (first_token < num_tokens)
|
||||
)
|
||||
safe_first_token = tl.maximum(first_token, 0)
|
||||
load_current_position = first_in_chunk & first_token_valid
|
||||
first_pos_t = tl.load(
|
||||
pos_ptr + safe_first_token * pos_stride_token,
|
||||
mask=load_current_position,
|
||||
other=0,
|
||||
)
|
||||
if IS_2D_POSITIONS:
|
||||
first_pos_h = tl.load(
|
||||
pos_ptr + pos_stride_axis + safe_first_token * pos_stride_token,
|
||||
mask=load_current_position,
|
||||
other=0,
|
||||
)
|
||||
first_pos_w = tl.load(
|
||||
pos_ptr
|
||||
+ 2 * pos_stride_axis
|
||||
+ safe_first_token * pos_stride_token,
|
||||
mask=load_current_position,
|
||||
other=0,
|
||||
)
|
||||
else:
|
||||
first_pos_h = first_pos_t
|
||||
first_pos_w = first_pos_t
|
||||
tail = (
|
||||
state_cache_ptr
|
||||
+ safe_state_block * state_cache_stride_block
|
||||
+ (first_position % STATE_SIZE) * state_cache_stride_token
|
||||
+ D
|
||||
).to(tl.pointer_type(tl.int64))
|
||||
load_cached_position = ~first_in_chunk & state_block_valid
|
||||
cached_pos_t = tl.load(tail, mask=load_cached_position, other=0)
|
||||
cached_pos_h = tl.load(tail + 1, mask=load_cached_position, other=0)
|
||||
cached_pos_w = tl.load(tail + 2, mask=load_cached_position, other=0)
|
||||
pos_t = tl.where(first_in_chunk, first_pos_t.to(tl.int64), cached_pos_t)
|
||||
pos_h = tl.where(first_in_chunk, first_pos_h.to(tl.int64), cached_pos_h)
|
||||
pos_w = tl.where(first_in_chunk, first_pos_w.to(tl.int64), cached_pos_w)
|
||||
else:
|
||||
pos_t = first_position
|
||||
pos_h = first_position
|
||||
pos_w = first_position
|
||||
y = _norm_rope(
|
||||
tl.reshape(pooled, (1, 1, D)),
|
||||
pos_t + tl.arange(0, 1),
|
||||
pos_h + tl.arange(0, 1),
|
||||
pos_w + tl.arange(0, 1),
|
||||
cos_sin_ptr,
|
||||
D // 2,
|
||||
k_norm_weight_ptr,
|
||||
eps,
|
||||
IS_K_MROPE,
|
||||
MROPE_H,
|
||||
MROPE_W,
|
||||
)
|
||||
compressed_block = (compressed_slot // COMP_PAGE_SIZE).to(tl.int64)
|
||||
compressed_row = compressed_slot % COMP_PAGE_SIZE
|
||||
tl.store(
|
||||
compressed_cache_ptr
|
||||
+ compressed_block * compressed_cache_stride_block
|
||||
+ compressed_row * compressed_cache_stride_token
|
||||
+ dims,
|
||||
tl.reshape(y, (D,)),
|
||||
mask=valid,
|
||||
)
|
||||
|
||||
if work_in_request == 0:
|
||||
# This CTA may have just read historical rows from the circular buffer.
|
||||
# Keep every lane past those loads before overwriting the same ring.
|
||||
tl.debug_barrier()
|
||||
# One CTA per request commits only the suffix retained by the ring.
|
||||
num_state_rows = tl.minimum(query_len, STATE_SIZE)
|
||||
for state_offset in tl.range(0, num_state_rows):
|
||||
token = query_end - num_state_rows + state_offset
|
||||
valid_token = (
|
||||
(token >= query_start) & (token < query_end) & (token < num_tokens)
|
||||
)
|
||||
slot = tl.load(state_slots_ptr + token, mask=valid_token, other=-1)
|
||||
valid_slot = (
|
||||
valid_token & (slot >= 0) & (slot < num_state_blocks * STATE_SIZE)
|
||||
)
|
||||
safe_slot = tl.maximum(slot, 0)
|
||||
state_row = (
|
||||
state_cache_ptr
|
||||
+ (safe_slot // STATE_SIZE).to(tl.int64) * state_cache_stride_block
|
||||
+ (safe_slot % STATE_SIZE) * state_cache_stride_token
|
||||
)
|
||||
k = tl.load(
|
||||
k_ptr + tl.maximum(token, 0) * k_stride_token + dims,
|
||||
mask=valid_slot,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(state_row + dims, k, mask=valid_slot)
|
||||
if CACHE_HAS_ROPE_POS:
|
||||
pos_t = tl.load(
|
||||
pos_ptr + tl.maximum(token, 0) * pos_stride_token,
|
||||
mask=valid_slot,
|
||||
other=0,
|
||||
)
|
||||
if IS_2D_POSITIONS:
|
||||
pos_h = tl.load(
|
||||
pos_ptr
|
||||
+ pos_stride_axis
|
||||
+ tl.maximum(token, 0) * pos_stride_token,
|
||||
mask=valid_slot,
|
||||
other=0,
|
||||
)
|
||||
pos_w = tl.load(
|
||||
pos_ptr
|
||||
+ 2 * pos_stride_axis
|
||||
+ tl.maximum(token, 0) * pos_stride_token,
|
||||
mask=valid_slot,
|
||||
other=0,
|
||||
)
|
||||
else:
|
||||
pos_h = pos_t
|
||||
pos_w = pos_t
|
||||
tail = (state_row + D).to(tl.pointer_type(tl.int64))
|
||||
tl.store(tail, pos_t.to(tl.int64), mask=valid_slot)
|
||||
tl.store(tail + 1, pos_h.to(tl.int64), mask=valid_slot)
|
||||
tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot)
|
||||
|
||||
|
||||
def qsa_pre_indexer(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
k_norm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
q_out: torch.Tensor,
|
||||
state_cache: torch.Tensor,
|
||||
state_slots: torch.Tensor,
|
||||
state_block_table: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
logical_positions: torch.Tensor,
|
||||
compressed_cache: torch.Tensor,
|
||||
compressed_slots: torch.Tensor,
|
||||
k_work_metadata: torch.Tensor,
|
||||
*,
|
||||
compress_ratio: int,
|
||||
mrope_section: tuple[int, int, int] | None,
|
||||
rope_pos_offset: int | None,
|
||||
) -> None:
|
||||
"""Normalize Q, compress K, then update the circular raw state."""
|
||||
num_tokens = q.shape[0]
|
||||
if num_tokens == 0:
|
||||
return
|
||||
num_q_heads, head_dim = q_out.shape[1:]
|
||||
assert cos_sin_cache.shape[-1] * 2 == head_dim
|
||||
assert q.shape == (num_tokens, num_q_heads * head_dim)
|
||||
assert k.shape == (num_tokens, head_dim)
|
||||
assert q.stride(-1) == 1
|
||||
assert k.stride(-1) == 1
|
||||
assert q_out.stride(-1) == 1
|
||||
assert cos_sin_cache.is_contiguous()
|
||||
assert state_cache.stride(-1) == 1
|
||||
assert compressed_cache.stride(-1) == 1
|
||||
assert k_work_metadata.ndim == 2 and k_work_metadata.shape[1] == 2
|
||||
is_2d_positions = positions.ndim == 2
|
||||
is_k_mrope = bool(mrope_section)
|
||||
cache_has_rope_pos = rope_pos_offset is not None
|
||||
assert rope_pos_offset is None or rope_pos_offset == head_dim
|
||||
if is_2d_positions:
|
||||
assert positions.shape == (3, num_tokens)
|
||||
assert is_k_mrope
|
||||
pos_stride_axis, pos_stride_token = positions.stride()
|
||||
else:
|
||||
assert positions.shape == (num_tokens,)
|
||||
pos_stride_axis, pos_stride_token = 0, positions.stride(0)
|
||||
section = mrope_section if mrope_section is not None else (0, 0, 0)
|
||||
assert len(section) == 3
|
||||
|
||||
# Swept on MI355X (64-lane wavefronts) over TILE_T_Q 1-16, TILE_H_Q 1-4 and
|
||||
# num_warps 1/2/4, from 4 to 16384 tokens: no config beat these by more
|
||||
# than noise, and 2 or 4 warps were slower everywhere.
|
||||
if num_tokens <= 4096:
|
||||
TILE_T_Q, TILE_H_Q = 2, 2
|
||||
else:
|
||||
TILE_T_Q, TILE_H_Q = 2, 4
|
||||
num_k_work = k_work_metadata.shape[0]
|
||||
num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q)
|
||||
_qsa_pre_indexer_kernel[(num_k_work + num_q_work,)](
|
||||
q,
|
||||
q.stride(0),
|
||||
k,
|
||||
k.stride(0),
|
||||
positions,
|
||||
pos_stride_axis,
|
||||
pos_stride_token,
|
||||
cos_sin_cache,
|
||||
q_norm_weight,
|
||||
k_norm_weight,
|
||||
eps,
|
||||
q_out,
|
||||
q_out.stride(0),
|
||||
q_out.stride(1),
|
||||
state_cache,
|
||||
state_cache.stride(0),
|
||||
state_cache.stride(1),
|
||||
state_slots,
|
||||
state_block_table,
|
||||
state_block_table.stride(0),
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
k_work_metadata,
|
||||
compressed_cache,
|
||||
compressed_cache.stride(0),
|
||||
compressed_cache.stride(1),
|
||||
num_tokens,
|
||||
state_cache.shape[0],
|
||||
compressed_cache.shape[0],
|
||||
num_k_work,
|
||||
HQ=num_q_heads,
|
||||
D=head_dim,
|
||||
TILE_T_Q=TILE_T_Q,
|
||||
TILE_H_Q=TILE_H_Q,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
STATE_SIZE=state_cache.shape[1],
|
||||
COMP_PAGE_SIZE=compressed_cache.shape[1],
|
||||
IS_2D_POSITIONS=is_2d_positions,
|
||||
IS_K_MROPE=is_k_mrope,
|
||||
CACHE_HAS_ROPE_POS=cache_has_rope_pos,
|
||||
MROPE_H=section[1],
|
||||
MROPE_W=section[2],
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["qsa_pre_indexer", "supports_fused_pre_indexer"]
|
||||
Reference in New Issue
Block a user