[Model] Qwen4Exp: fp8_e4m3 main KV cache on the QSA path (#55557)
Signed-off-by: Andrea Semeraro <andrea.semeraro@sezione1.it> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Claude
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Jiangyun Zhu
parent
e97ff80613
commit
dff1bde84d
@@ -203,7 +203,14 @@ def _qsa_sparse_paged_attention_reference(
|
||||
block_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
k_scale: float = 1.0,
|
||||
v_scale: float = 1.0,
|
||||
) -> torch.Tensor:
|
||||
"""Dense reference for QSA sparse paged attention.
|
||||
|
||||
Mirrors the kernel's dequant: fp8-e4m3 K/V caches are dequantized with the
|
||||
per-tensor k_scale/v_scale host floats; bf16 caches use unit scales.
|
||||
"""
|
||||
output = torch.zeros_like(q)
|
||||
repeats = q.shape[1] // k_cache.shape[2]
|
||||
page_size = k_cache.shape[1]
|
||||
@@ -215,13 +222,15 @@ def _qsa_sparse_paged_attention_reference(
|
||||
request = token_to_req[row].long()
|
||||
pages = block_table[request, logical // page_size].long()
|
||||
offsets = logical % page_size
|
||||
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
|
||||
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
|
||||
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
|
||||
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
|
||||
q.dtype
|
||||
keys = (k_cache[pages, offsets].float() * k_scale).repeat_interleave(
|
||||
repeats, dim=1
|
||||
)
|
||||
values = (v_cache[pages, offsets].float() * v_scale).repeat_interleave(
|
||||
repeats, dim=1
|
||||
)
|
||||
scores = torch.einsum("hd,khd->hk", q[row].float(), keys)
|
||||
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
|
||||
output[row] = torch.einsum("hk,khd->hd", probabilities, values).to(q.dtype)
|
||||
return output
|
||||
|
||||
|
||||
@@ -906,22 +915,28 @@ def test_qsa_block_expansion_correctness() -> None:
|
||||
"page_size",
|
||||
"use_prefill_config",
|
||||
"num_requests",
|
||||
"fp8",
|
||||
),
|
||||
[
|
||||
# Production page sizes from hybrid-cache block alignment: 784/800
|
||||
# at TP4 and 1568/1600 at TP1/TP2 (no-MTP / MTP num_spec=3). Head
|
||||
# splits are per-rank TP1/TP2/TP4; the largest batch runs both
|
||||
# use_prefill_config variants.
|
||||
pytest.param(1, 24, 2, 1600, True, 2, id="tp1_r1"),
|
||||
pytest.param(16, 12, 1, 1600, True, 3, id="tp2_r16"),
|
||||
pytest.param(32, 6, 1, 800, True, 5, id="tp4_r32"),
|
||||
pytest.param(128, 24, 2, 1568, True, 7, id="tp1_r128"),
|
||||
pytest.param(257, 6, 1, 800, True, 13, id="tp4_r257"),
|
||||
pytest.param(513, 6, 1, 784, True, 17, id="tp4_r513"),
|
||||
pytest.param(700, 6, 1, 800, True, 23, id="tp4_r700"),
|
||||
pytest.param(1024, 24, 2, 1600, True, 33, id="tp1_r1024"),
|
||||
pytest.param(2048, 24, 2, 1600, True, 63, id="tp1_r2048_prefill"),
|
||||
pytest.param(2048, 24, 2, 1600, False, 63, id="tp1_r2048_uniform"),
|
||||
pytest.param(1, 24, 2, 1600, True, 2, False, id="tp1_r1"),
|
||||
pytest.param(16, 12, 1, 1600, True, 3, False, id="tp2_r16"),
|
||||
pytest.param(32, 6, 1, 800, True, 5, False, id="tp4_r32"),
|
||||
pytest.param(128, 24, 2, 1568, True, 7, False, id="tp1_r128"),
|
||||
pytest.param(257, 6, 1, 800, True, 13, False, id="tp4_r257"),
|
||||
pytest.param(513, 6, 1, 784, True, 17, False, id="tp4_r513"),
|
||||
pytest.param(700, 6, 1, 800, True, 23, False, id="tp4_r700"),
|
||||
pytest.param(1024, 24, 2, 1600, True, 33, False, id="tp1_r1024"),
|
||||
pytest.param(2048, 24, 2, 1600, True, 63, False, id="tp1_r2048_prefill"),
|
||||
pytest.param(2048, 24, 2, 1600, False, 63, False, id="tp1_r2048_uniform"),
|
||||
# fp8_e4m3 K/V caches on the TP1 head split.
|
||||
pytest.param(1, 24, 2, 1600, True, 2, True, id="tp1_r1_fp8"),
|
||||
pytest.param(128, 24, 2, 1568, True, 7, True, id="tp1_r128_fp8"),
|
||||
pytest.param(2048, 24, 2, 1600, True, 63, True, id="tp1_r2048_prefill_fp8"),
|
||||
pytest.param(2048, 24, 2, 1600, False, 63, True, id="tp1_r2048_uniform_fp8"),
|
||||
],
|
||||
)
|
||||
def test_qsa_sparse_paged_attention_correctness(
|
||||
@@ -931,8 +946,18 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
page_size: int,
|
||||
use_prefill_config: bool,
|
||||
num_requests: int,
|
||||
fp8: bool,
|
||||
) -> None:
|
||||
"""QSA sparse paged attention matches the dense reference.
|
||||
|
||||
fp8 only changes the K/V cache dtype (e4m3 with a per-tensor scale pair) and
|
||||
the scales; the reference dequantizes the same cache with those scales, so
|
||||
both paths compare the production kernel against the reference on identical
|
||||
inputs. fp8=True additionally covers the host-side scale folding.
|
||||
"""
|
||||
torch.manual_seed(2)
|
||||
# One QSA attention problem: bf16 Q and paged K/V, a packed selection with
|
||||
# the trailing count column, block table and row-to-request map.
|
||||
head_dim = 256
|
||||
num_selected_pages = 64
|
||||
# Keep the newest page outside the synthetic top-k as causal headroom.
|
||||
@@ -1013,8 +1038,18 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
indexer_budget,
|
||||
logical_indices,
|
||||
)
|
||||
assert logical_indices.shape == (num_rows, selection_width + 1)
|
||||
scale = q.shape[-1] ** -0.5
|
||||
|
||||
scale = head_dim**-0.5
|
||||
|
||||
if fp8:
|
||||
# A fixed non-unit pair (k != v) exercises the host-side scale folding
|
||||
# and catches a k/v swap; scales are host floats, as the layer exposes
|
||||
# them. Stored values are the scaled ones, as reshape_and_cache does.
|
||||
k_scale, v_scale = 0.5, 2.0
|
||||
k_cache = (k_cache / k_scale).to(torch.float8_e4m3fn)
|
||||
v_cache = (v_cache / v_scale).to(torch.float8_e4m3fn)
|
||||
else:
|
||||
k_scale, v_scale = 1.0, 1.0
|
||||
|
||||
actual = qsa_ops.qsa_sparse_paged_attention(
|
||||
q,
|
||||
@@ -1024,6 +1059,8 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
block_table,
|
||||
token_to_req,
|
||||
use_prefill_config=use_prefill_config,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
output_gate=output_gate,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
@@ -1034,6 +1071,8 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
block_table,
|
||||
token_to_req,
|
||||
scale,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
expected = expected * torch.sigmoid(output_gate)
|
||||
|
||||
|
||||
@@ -4,15 +4,24 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
TritonWarmupTensor,
|
||||
triton_scalar_specialization_rep,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _is_sm120() -> bool:
|
||||
"""True on sm_120 (RTX PRO 6000 Blackwell): selects the sm_120 tuning table."""
|
||||
return current_platform.get_device_capability() == (12, 0)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_rows", "num_requests"])
|
||||
def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
q_ptr,
|
||||
@@ -24,6 +33,8 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
partial_output_ptr,
|
||||
partial_lse_ptr,
|
||||
output_ptr,
|
||||
softmax_scale,
|
||||
output_scale,
|
||||
output_gate_ptr,
|
||||
stride_q_row,
|
||||
stride_q_head,
|
||||
@@ -52,6 +63,7 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
NUM_TILES: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
IS_FP8: tl.constexpr,
|
||||
) -> None:
|
||||
row = tl.program_id(0)
|
||||
kv_head = tl.program_id(1)
|
||||
@@ -81,7 +93,10 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
max_value = tl.full((BLOCK_M,), -1.0e20, dtype=tl.float32)
|
||||
normalizer = tl.zeros((BLOCK_M,), dtype=tl.float32)
|
||||
accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32)
|
||||
softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634
|
||||
# softmax_scale is the host-side attention scale (1/sqrt(head_dim), with the
|
||||
# fp8 K dequant scale already pre-multiplied in); convert to log2 units once
|
||||
# here for the exp2-based online softmax.
|
||||
score_scale = softmax_scale * 1.4426950408889634
|
||||
|
||||
tile_end = tl.minimum(NUM_TILES, tl.cdiv(tl.minimum(valid_count, TOPK), BLOCK_N))
|
||||
|
||||
@@ -129,15 +144,26 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
if IS_FP8:
|
||||
# e4m3 -> Q dtype is exact; keep the QK dot in Q's dtype (fp8 QK
|
||||
# measured slower here and less accurate).
|
||||
keys = keys.to(query.dtype)
|
||||
scores = tl.dot(query, keys)
|
||||
# Scaling scores avoids re-quantizing a scaled query to BF16.
|
||||
scores *= softmax_scale_log2
|
||||
# Scaling scores avoids re-quantizing a scaled query to BF16; for fp8
|
||||
# caches the K dequant scale is already folded into softmax_scale on the
|
||||
# host.
|
||||
scores *= score_scale
|
||||
scores = tl.where(valid[None, :], scores, -1.0e20)
|
||||
next_max = tl.maximum(max_value, tl.max(scores, axis=1))
|
||||
alpha = tl.math.exp2(max_value - next_max)
|
||||
probabilities = tl.where(
|
||||
valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0
|
||||
)
|
||||
if IS_FP8:
|
||||
# Dequant V to fp16 (not bf16) for the PV dot: P <= 1 (online
|
||||
# softmax) so fp16 has the range, its wider mantissa is more
|
||||
# accurate, and the fp8->fp16 upcast with an fp16 PV dot is faster.
|
||||
values = values.to(tl.float16)
|
||||
accumulator = tl.dot(
|
||||
probabilities.to(values.dtype),
|
||||
values,
|
||||
@@ -147,9 +173,14 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
max_value = next_max
|
||||
|
||||
has_values = normalizer > 0
|
||||
# Fold the fp8 V dequant scale (output_scale, 1.0 for bf16) into a per-row
|
||||
# reciprocal normalizer, so the output is a per-row multiply rather than a
|
||||
# HEAD_DIM-wide scale. The split-K LSE below keeps the unscaled normalizer,
|
||||
# so the merge stays correct.
|
||||
inv_normalizer = output_scale / tl.maximum(normalizer, 1.0e-20)
|
||||
normalized_output = tl.where(
|
||||
has_values[:, None],
|
||||
accumulator / tl.maximum(normalizer[:, None], 1.0e-20),
|
||||
accumulator * inv_normalizer[:, None],
|
||||
0.0,
|
||||
)
|
||||
output_mask = head_offsets[:, None] < GROUP_SIZE
|
||||
@@ -439,18 +470,65 @@ def _compress_qsa_groups_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _select_sm120_config(
|
||||
base_programs: int, use_prefill_config: bool, is_fp8: bool
|
||||
) -> tuple[int, int, int]:
|
||||
"""(block_n, target_splits, num_warps) retuned on sm_120 (RTX PRO 6000
|
||||
Blackwell), split by cache dtype: bf16 and fp8 favour different configs on
|
||||
sm_120, most visibly on the large-prefill region. Each entry is the fastest
|
||||
config that stays correct on its own path; the main win is more warps."""
|
||||
if is_fp8:
|
||||
if base_programs > 2048:
|
||||
return (32, 2, 1) if use_prefill_config else (64, 1, 2)
|
||||
if base_programs <= 24:
|
||||
return 64, 64, 8
|
||||
if base_programs <= 32:
|
||||
return 128, 8, 4
|
||||
if base_programs <= 64:
|
||||
return 64, 8, 4
|
||||
if base_programs <= 128:
|
||||
return 32, 4, 4
|
||||
if base_programs <= 256:
|
||||
return 128, 4, 4
|
||||
if base_programs <= 512:
|
||||
return 64, 4, 4
|
||||
return 32, 2, 1
|
||||
# bf16 K/V cache.
|
||||
if base_programs > 2048:
|
||||
return 64, 2, 2
|
||||
if base_programs <= 24:
|
||||
return 64, 64, 8
|
||||
if base_programs <= 64:
|
||||
return 64, 8, 8
|
||||
if base_programs <= 128:
|
||||
return 32, 8, 8
|
||||
if base_programs <= 256:
|
||||
return 32, 8, 1
|
||||
if base_programs <= 512:
|
||||
return 64, 4, 2
|
||||
return 32, 4, 1
|
||||
|
||||
|
||||
def _select_config(
|
||||
num_rows: int, num_kv_heads: int, use_prefill_config: bool, num_columns: int
|
||||
num_rows: int,
|
||||
num_kv_heads: int,
|
||||
use_prefill_config: bool,
|
||||
num_columns: int,
|
||||
is_fp8: bool = False,
|
||||
) -> tuple[int, int, int, int]:
|
||||
"""Select (block_n, num_warps, num_tiles, num_splits) for the kernel.
|
||||
|
||||
Tuned on GB300 for the Qwen3.8-Flash-Next TP1/TP2/TP4 shapes, keyed on
|
||||
base_programs = num_rows * num_kv_heads. The bp > 2048 region splits on
|
||||
use_prefill_config (capture-stable: at FULL-graph capture max_query_len is the
|
||||
uniform decode/verify length).
|
||||
Keyed on base_programs = num_rows * num_kv_heads. The bp > 2048 region splits
|
||||
on use_prefill_config (capture-stable: at FULL-graph capture max_query_len is
|
||||
the uniform decode/verify length). This default table was tuned on GB300;
|
||||
sm_120 (RTX PRO 6000 Blackwell) dispatches to _select_sm120_config instead.
|
||||
"""
|
||||
base_programs = num_rows * num_kv_heads
|
||||
if base_programs > 2048:
|
||||
if _is_sm120():
|
||||
BLOCK_N, target_splits, num_warps = _select_sm120_config(
|
||||
base_programs, use_prefill_config, is_fp8
|
||||
)
|
||||
elif base_programs > 2048:
|
||||
BLOCK_N, target_splits, num_warps = (
|
||||
(32, 1, 1) if use_prefill_config else (64, 1, 2)
|
||||
)
|
||||
@@ -483,10 +561,17 @@ def qsa_sparse_paged_attention(
|
||||
token_to_req: torch.Tensor,
|
||||
use_prefill_config: bool,
|
||||
out: torch.Tensor | None = None,
|
||||
k_scale: float | None = None,
|
||||
v_scale: float | None = None,
|
||||
*,
|
||||
output_gate: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Run sparse GQA directly over paged BF16 K/V caches.
|
||||
"""Run sparse GQA directly over paged BF16 or FP8-e4m3 K/V caches.
|
||||
|
||||
With fp8 caches, k_scale/v_scale are the layer's per-tensor dequant scales
|
||||
as host floats (e.g. layer._k_scale_float/_v_scale_float): k_scale is
|
||||
pre-multiplied into the softmax scale and v_scale becomes the kernel's
|
||||
output scale, so the kernel needs no device scale buffers.
|
||||
|
||||
logical_indices is the PACKED selection buffer: [rows, selection_width + 1]
|
||||
with the trailing column holding each row's valid-entry count (written by
|
||||
@@ -510,7 +595,19 @@ def qsa_sparse_paged_attention(
|
||||
raise ValueError("QSA sparse attention requires valid grouped-query heads")
|
||||
head_dim = q.shape[2]
|
||||
assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0
|
||||
assert q.dtype == k_cache.dtype == v_cache.dtype == torch.bfloat16
|
||||
assert q.dtype == torch.bfloat16
|
||||
assert k_cache.dtype == v_cache.dtype
|
||||
is_fp8 = k_cache.dtype == torch.float8_e4m3fn
|
||||
if is_fp8:
|
||||
assert k_scale is not None and v_scale is not None
|
||||
# Host pre-multiply: fold the K dequant scale into the attention scale
|
||||
# and pass V's dequant scale as the kernel's output scale.
|
||||
softmax_scale = (head_dim**-0.5) * float(k_scale)
|
||||
output_scale = float(v_scale)
|
||||
else:
|
||||
assert k_cache.dtype == torch.bfloat16
|
||||
softmax_scale = head_dim**-0.5
|
||||
output_scale = 1.0
|
||||
assert logical_indices.dtype == block_table.dtype == torch.int32
|
||||
assert token_to_req.dtype == torch.int32
|
||||
assert q.device == k_cache.device == v_cache.device
|
||||
@@ -535,7 +632,7 @@ def qsa_sparse_paged_attention(
|
||||
block_m = triton.next_power_of_2(group_size)
|
||||
selection_width = logical_indices.shape[1] - 1 # trailing column is the count
|
||||
block_n, partial_warps, num_tiles, num_splits = _select_config(
|
||||
q.shape[0], k_cache.shape[2], use_prefill_config, selection_width
|
||||
q.shape[0], k_cache.shape[2], use_prefill_config, selection_width, is_fp8
|
||||
)
|
||||
|
||||
# Split=1 writes output directly and compiles out all workspace accesses.
|
||||
@@ -565,6 +662,8 @@ def qsa_sparse_paged_attention(
|
||||
partial_output,
|
||||
partial_lse,
|
||||
out,
|
||||
softmax_scale,
|
||||
output_scale,
|
||||
output_gate_view,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
@@ -593,6 +692,7 @@ def qsa_sparse_paged_attention(
|
||||
NUM_TILES=num_tiles,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=block_n,
|
||||
IS_FP8=is_fp8,
|
||||
num_warps=partial_warps,
|
||||
num_stages=2,
|
||||
)
|
||||
@@ -630,13 +730,18 @@ def warmup_qsa_sparse_paged_attention(
|
||||
|
||||
head_dim = kv_cache.shape[-1] // 2
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_dim, dim=-1)
|
||||
# An fp8 cache is allocated as uint8 and viewed as e4m3 at attention time.
|
||||
is_fp8 = kv_cache.dtype == torch.uint8
|
||||
cache_dtype = torch.float8_e4m3fn if is_fp8 else key_cache.dtype
|
||||
num_kv_heads = key_cache.shape[2]
|
||||
group_size = num_query_heads // num_kv_heads
|
||||
block_m = triton.next_power_of_2(group_size)
|
||||
|
||||
# Every config the dispatch can pick for this group size.
|
||||
profiles = {
|
||||
_select_config(num_rows, num_kv_heads, use_prefill_config, selection_width)
|
||||
_select_config(
|
||||
num_rows, num_kv_heads, use_prefill_config, selection_width, is_fp8
|
||||
)
|
||||
for num_rows in range(1, 8193)
|
||||
for use_prefill_config in (False, True)
|
||||
}
|
||||
@@ -650,12 +755,12 @@ def warmup_qsa_sparse_paged_attention(
|
||||
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
|
||||
)
|
||||
k_cache_ptr = TritonWarmupTensor(
|
||||
key_cache.dtype,
|
||||
cache_dtype,
|
||||
shape=tuple(key_cache.shape),
|
||||
strides=tuple(key_cache.stride()),
|
||||
)
|
||||
v_cache_ptr = TritonWarmupTensor(
|
||||
value_cache.dtype,
|
||||
cache_dtype,
|
||||
shape=tuple(value_cache.shape),
|
||||
strides=tuple(value_cache.stride()),
|
||||
)
|
||||
@@ -701,6 +806,8 @@ def warmup_qsa_sparse_paged_attention(
|
||||
partial_output_ptr,
|
||||
partial_lse_ptr,
|
||||
output_ptr,
|
||||
1.0,
|
||||
1.0,
|
||||
output_gate_ptr,
|
||||
row_stride,
|
||||
head_stride,
|
||||
@@ -729,6 +836,7 @@ def warmup_qsa_sparse_paged_attention(
|
||||
NUM_TILES=num_tiles,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=block_n,
|
||||
IS_FP8=is_fp8,
|
||||
num_warps=warps,
|
||||
num_stages=2,
|
||||
grid=(num_rows, num_kv_heads, num_splits),
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
|
||||
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.transformers_utils.configs.qwen4_exp import (
|
||||
Qwen4ExpTextConfig,
|
||||
)
|
||||
@@ -64,7 +65,38 @@ class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
|
||||
"""FullAttentionSpec backend used by the merged QSA owner."""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
|
||||
# fp8/fp8_e4m3: e4m3 bytes in a uint8 cache, written by reshape_and_cache
|
||||
# with the layer's per-tensor scales and dequantized on load inside the QSA
|
||||
# Triton kernel. flash-attn never runs over this cache, so its fp8 probe
|
||||
# does not apply (see supports_kv_cache_dtype and the impl constructor).
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
||||
return kv_cache_dtype is None or kv_cache_dtype in cls.supported_kv_cache_dtypes
|
||||
|
||||
@classmethod
|
||||
def supports_combination(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType | None,
|
||||
block_size: int | None,
|
||||
use_mla: bool,
|
||||
has_sink: bool,
|
||||
use_sparse: bool,
|
||||
use_mm_prefix: bool,
|
||||
device_capability: DeviceCapability,
|
||||
) -> str | None:
|
||||
# QSA dequantizes the fp8 KV in its own Triton kernel and never runs
|
||||
# flash-attn over the quantized cache, so the parent's fp8-KV rejection
|
||||
# does not apply and every combination it is handed is accepted here.
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
@@ -98,16 +130,53 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
supports_dcp: bool = False
|
||||
supports_pcp: bool = False
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int,
|
||||
alibi_slopes: list[float] | None,
|
||||
sliding_window: int | None,
|
||||
kv_cache_dtype: str,
|
||||
logits_soft_cap: float | None = None,
|
||||
attn_type: AttentionType = AttentionType.DECODER,
|
||||
kv_sharing_target_layer_name: str | None = None,
|
||||
sinks: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
# The parent constructor probes flash-attn for quantized-KV support and
|
||||
# raises where it is unavailable (sm120), but QSA dequantizes fp8 inside
|
||||
# its own Triton kernel and never runs flash-attn over the cache. Hand
|
||||
# the parent "auto" for that probe and restore the real dtype afterwards:
|
||||
# the parent only uses it there, and do_kv_cache_update reads the
|
||||
# attribute at call time.
|
||||
real_kv_cache_dtype = kv_cache_dtype
|
||||
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
kv_cache_dtype = "auto"
|
||||
super().__init__(
|
||||
num_heads,
|
||||
head_size,
|
||||
scale,
|
||||
num_kv_heads,
|
||||
alibi_slopes,
|
||||
sliding_window,
|
||||
kv_cache_dtype,
|
||||
logits_soft_cap,
|
||||
attn_type,
|
||||
kv_sharing_target_layer_name,
|
||||
sinks,
|
||||
)
|
||||
self.kv_cache_dtype = real_kv_cache_dtype
|
||||
if not is_flash_attn_varlen_func_available():
|
||||
raise NotImplementedError("Qwen4Exp QSA requires FlashAttention")
|
||||
if self.dcp_world_size != 1:
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA does not support decode context parallelism"
|
||||
)
|
||||
if self.kv_cache_dtype not in ("auto", "bfloat16"):
|
||||
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
|
||||
if self.kv_cache_dtype not in ("auto", "bfloat16", "fp8", "fp8_e4m3"):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA requires a BF16 or FP8-e4m3 main KV cache"
|
||||
)
|
||||
self.supports_quant_query_input = False
|
||||
|
||||
def forward_qsa(
|
||||
@@ -144,8 +213,23 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
logical_indices = topk_buffer[:num_tokens]
|
||||
token_to_req = token_to_req[:num_tokens]
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V")
|
||||
k_scale = v_scale = None
|
||||
if self.kv_cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
# The cache is allocated as uint8; reinterpret the e4m3 bytes
|
||||
# (same itemsize, so shape and strides are preserved).
|
||||
key_cache = key_cache.view(torch.float8_e4m3fn)
|
||||
value_cache = value_cache.view(torch.float8_e4m3fn)
|
||||
# Host-side per-tensor dequant scales (Python floats), as used by
|
||||
# other host-scale backends; folded into the kernel's scales.
|
||||
k_scale = layer._k_scale_float
|
||||
v_scale = layer._v_scale_float
|
||||
if query.dtype != torch.bfloat16 or key_cache.dtype not in (
|
||||
torch.bfloat16,
|
||||
torch.float8_e4m3fn,
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA requires BF16 Q and BF16 or FP8-e4m3 K/V"
|
||||
)
|
||||
|
||||
from .ops.qsa import qsa_sparse_paged_attention
|
||||
|
||||
@@ -158,6 +242,8 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
token_to_req,
|
||||
use_prefill_config,
|
||||
output[:num_tokens],
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
output_gate=output_gate[:num_tokens],
|
||||
)
|
||||
return output
|
||||
@@ -185,8 +271,15 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
|
||||
if model_config.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
|
||||
if cache_config.cache_dtype not in ("auto", "bfloat16"):
|
||||
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
|
||||
if cache_config.cache_dtype not in (
|
||||
"auto",
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA requires a BF16 or FP8-e4m3 main KV cache"
|
||||
)
|
||||
if getattr(quant_config, "kv_cache_scheme", None) is not None:
|
||||
raise NotImplementedError("Qwen4Exp QSA does not support KV quantization")
|
||||
parallel_config = vllm_config.parallel_config
|
||||
@@ -285,8 +378,10 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
|
||||
self.kv_cache_dtype, model_config
|
||||
)
|
||||
if self.kv_cache_torch_dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA requires BF16 cache storage")
|
||||
if self.kv_cache_torch_dtype not in (torch.bfloat16, torch.uint8):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp QSA requires BF16 or FP8-e4m3 (uint8) cache storage"
|
||||
)
|
||||
self.kv_sharing_target_layer_name = None
|
||||
self.kv_cache = torch.tensor([])
|
||||
set_default_quant_scales(self, register_buffer=True)
|
||||
|
||||
Reference in New Issue
Block a user