[Model] Qwen4Exp: fp8_e4m3 main KV cache on the QSA path (#55557)

Signed-off-by: Andrea Semeraro <andrea.semeraro@sezione1.it>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
semerandre
2026-09-16 15:07:52 +00:00
committed by GitHub
co-authored by Claude mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Jiangyun Zhu
parent e97ff80613
commit dff1bde84d
3 changed files with 287 additions and 45 deletions
+57 -18
View File
@@ -203,7 +203,14 @@ def _qsa_sparse_paged_attention_reference(
block_table: torch.Tensor,
token_to_req: torch.Tensor,
softmax_scale: float,
k_scale: float = 1.0,
v_scale: float = 1.0,
) -> torch.Tensor:
"""Dense reference for QSA sparse paged attention.
Mirrors the kernel's dequant: fp8-e4m3 K/V caches are dequantized with the
per-tensor k_scale/v_scale host floats; bf16 caches use unit scales.
"""
output = torch.zeros_like(q)
repeats = q.shape[1] // k_cache.shape[2]
page_size = k_cache.shape[1]
@@ -215,13 +222,15 @@ def _qsa_sparse_paged_attention_reference(
request = token_to_req[row].long()
pages = block_table[request, logical // page_size].long()
offsets = logical % page_size
keys = k_cache[pages, offsets].repeat_interleave(repeats, dim=1)
values = v_cache[pages, offsets].repeat_interleave(repeats, dim=1)
scores = torch.einsum("hd,khd->hk", q[row].float(), keys.float())
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
output[row] = torch.einsum("hk,khd->hd", probabilities, values.float()).to(
q.dtype
keys = (k_cache[pages, offsets].float() * k_scale).repeat_interleave(
repeats, dim=1
)
values = (v_cache[pages, offsets].float() * v_scale).repeat_interleave(
repeats, dim=1
)
scores = torch.einsum("hd,khd->hk", q[row].float(), keys)
probabilities = torch.softmax(scores * softmax_scale, dim=-1)
output[row] = torch.einsum("hk,khd->hd", probabilities, values).to(q.dtype)
return output
@@ -906,22 +915,28 @@ def test_qsa_block_expansion_correctness() -> None:
"page_size",
"use_prefill_config",
"num_requests",
"fp8",
),
[
# Production page sizes from hybrid-cache block alignment: 784/800
# at TP4 and 1568/1600 at TP1/TP2 (no-MTP / MTP num_spec=3). Head
# splits are per-rank TP1/TP2/TP4; the largest batch runs both
# use_prefill_config variants.
pytest.param(1, 24, 2, 1600, True, 2, id="tp1_r1"),
pytest.param(16, 12, 1, 1600, True, 3, id="tp2_r16"),
pytest.param(32, 6, 1, 800, True, 5, id="tp4_r32"),
pytest.param(128, 24, 2, 1568, True, 7, id="tp1_r128"),
pytest.param(257, 6, 1, 800, True, 13, id="tp4_r257"),
pytest.param(513, 6, 1, 784, True, 17, id="tp4_r513"),
pytest.param(700, 6, 1, 800, True, 23, id="tp4_r700"),
pytest.param(1024, 24, 2, 1600, True, 33, id="tp1_r1024"),
pytest.param(2048, 24, 2, 1600, True, 63, id="tp1_r2048_prefill"),
pytest.param(2048, 24, 2, 1600, False, 63, id="tp1_r2048_uniform"),
pytest.param(1, 24, 2, 1600, True, 2, False, id="tp1_r1"),
pytest.param(16, 12, 1, 1600, True, 3, False, id="tp2_r16"),
pytest.param(32, 6, 1, 800, True, 5, False, id="tp4_r32"),
pytest.param(128, 24, 2, 1568, True, 7, False, id="tp1_r128"),
pytest.param(257, 6, 1, 800, True, 13, False, id="tp4_r257"),
pytest.param(513, 6, 1, 784, True, 17, False, id="tp4_r513"),
pytest.param(700, 6, 1, 800, True, 23, False, id="tp4_r700"),
pytest.param(1024, 24, 2, 1600, True, 33, False, id="tp1_r1024"),
pytest.param(2048, 24, 2, 1600, True, 63, False, id="tp1_r2048_prefill"),
pytest.param(2048, 24, 2, 1600, False, 63, False, id="tp1_r2048_uniform"),
# fp8_e4m3 K/V caches on the TP1 head split.
pytest.param(1, 24, 2, 1600, True, 2, True, id="tp1_r1_fp8"),
pytest.param(128, 24, 2, 1568, True, 7, True, id="tp1_r128_fp8"),
pytest.param(2048, 24, 2, 1600, True, 63, True, id="tp1_r2048_prefill_fp8"),
pytest.param(2048, 24, 2, 1600, False, 63, True, id="tp1_r2048_uniform_fp8"),
],
)
def test_qsa_sparse_paged_attention_correctness(
@@ -931,8 +946,18 @@ def test_qsa_sparse_paged_attention_correctness(
page_size: int,
use_prefill_config: bool,
num_requests: int,
fp8: bool,
) -> None:
"""QSA sparse paged attention matches the dense reference.
fp8 only changes the K/V cache dtype (e4m3 with a per-tensor scale pair) and
the scales; the reference dequantizes the same cache with those scales, so
both paths compare the production kernel against the reference on identical
inputs. fp8=True additionally covers the host-side scale folding.
"""
torch.manual_seed(2)
# One QSA attention problem: bf16 Q and paged K/V, a packed selection with
# the trailing count column, block table and row-to-request map.
head_dim = 256
num_selected_pages = 64
# Keep the newest page outside the synthetic top-k as causal headroom.
@@ -1013,8 +1038,18 @@ def test_qsa_sparse_paged_attention_correctness(
indexer_budget,
logical_indices,
)
assert logical_indices.shape == (num_rows, selection_width + 1)
scale = q.shape[-1] ** -0.5
scale = head_dim**-0.5
if fp8:
# A fixed non-unit pair (k != v) exercises the host-side scale folding
# and catches a k/v swap; scales are host floats, as the layer exposes
# them. Stored values are the scaled ones, as reshape_and_cache does.
k_scale, v_scale = 0.5, 2.0
k_cache = (k_cache / k_scale).to(torch.float8_e4m3fn)
v_cache = (v_cache / v_scale).to(torch.float8_e4m3fn)
else:
k_scale, v_scale = 1.0, 1.0
actual = qsa_ops.qsa_sparse_paged_attention(
q,
@@ -1024,6 +1059,8 @@ def test_qsa_sparse_paged_attention_correctness(
block_table,
token_to_req,
use_prefill_config=use_prefill_config,
k_scale=k_scale,
v_scale=v_scale,
output_gate=output_gate,
)
expected = _qsa_sparse_paged_attention_reference(
@@ -1034,6 +1071,8 @@ def test_qsa_sparse_paged_attention_correctness(
block_table,
token_to_req,
scale,
k_scale=k_scale,
v_scale=v_scale,
)
expected = expected * torch.sigmoid(output_gate)
+124 -16
View File
@@ -4,15 +4,24 @@
from __future__ import annotations
from functools import lru_cache
import torch
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
TritonWarmupTensor,
triton_scalar_specialization_rep,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, tl, triton
@lru_cache(maxsize=1)
def _is_sm120() -> bool:
"""True on sm_120 (RTX PRO 6000 Blackwell): selects the sm_120 tuning table."""
return current_platform.get_device_capability() == (12, 0)
@triton.jit(do_not_specialize=["num_rows", "num_requests"])
def _qsa_sparse_paged_gqa_splitk_kernel(
q_ptr,
@@ -24,6 +33,8 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
partial_output_ptr,
partial_lse_ptr,
output_ptr,
softmax_scale,
output_scale,
output_gate_ptr,
stride_q_row,
stride_q_head,
@@ -52,6 +63,7 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
NUM_TILES: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
IS_FP8: tl.constexpr,
) -> None:
row = tl.program_id(0)
kv_head = tl.program_id(1)
@@ -81,7 +93,10 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
max_value = tl.full((BLOCK_M,), -1.0e20, dtype=tl.float32)
normalizer = tl.zeros((BLOCK_M,), dtype=tl.float32)
accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32)
softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634
# softmax_scale is the host-side attention scale (1/sqrt(head_dim), with the
# fp8 K dequant scale already pre-multiplied in); convert to log2 units once
# here for the exp2-based online softmax.
score_scale = softmax_scale * 1.4426950408889634
tile_end = tl.minimum(NUM_TILES, tl.cdiv(tl.minimum(valid_count, TOPK), BLOCK_N))
@@ -129,15 +144,26 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
mask=valid[:, None],
other=0.0,
)
if IS_FP8:
# e4m3 -> Q dtype is exact; keep the QK dot in Q's dtype (fp8 QK
# measured slower here and less accurate).
keys = keys.to(query.dtype)
scores = tl.dot(query, keys)
# Scaling scores avoids re-quantizing a scaled query to BF16.
scores *= softmax_scale_log2
# Scaling scores avoids re-quantizing a scaled query to BF16; for fp8
# caches the K dequant scale is already folded into softmax_scale on the
# host.
scores *= score_scale
scores = tl.where(valid[None, :], scores, -1.0e20)
next_max = tl.maximum(max_value, tl.max(scores, axis=1))
alpha = tl.math.exp2(max_value - next_max)
probabilities = tl.where(
valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0
)
if IS_FP8:
# Dequant V to fp16 (not bf16) for the PV dot: P <= 1 (online
# softmax) so fp16 has the range, its wider mantissa is more
# accurate, and the fp8->fp16 upcast with an fp16 PV dot is faster.
values = values.to(tl.float16)
accumulator = tl.dot(
probabilities.to(values.dtype),
values,
@@ -147,9 +173,14 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
max_value = next_max
has_values = normalizer > 0
# Fold the fp8 V dequant scale (output_scale, 1.0 for bf16) into a per-row
# reciprocal normalizer, so the output is a per-row multiply rather than a
# HEAD_DIM-wide scale. The split-K LSE below keeps the unscaled normalizer,
# so the merge stays correct.
inv_normalizer = output_scale / tl.maximum(normalizer, 1.0e-20)
normalized_output = tl.where(
has_values[:, None],
accumulator / tl.maximum(normalizer[:, None], 1.0e-20),
accumulator * inv_normalizer[:, None],
0.0,
)
output_mask = head_offsets[:, None] < GROUP_SIZE
@@ -439,18 +470,65 @@ def _compress_qsa_groups_kernel(
)
def _select_sm120_config(
base_programs: int, use_prefill_config: bool, is_fp8: bool
) -> tuple[int, int, int]:
"""(block_n, target_splits, num_warps) retuned on sm_120 (RTX PRO 6000
Blackwell), split by cache dtype: bf16 and fp8 favour different configs on
sm_120, most visibly on the large-prefill region. Each entry is the fastest
config that stays correct on its own path; the main win is more warps."""
if is_fp8:
if base_programs > 2048:
return (32, 2, 1) if use_prefill_config else (64, 1, 2)
if base_programs <= 24:
return 64, 64, 8
if base_programs <= 32:
return 128, 8, 4
if base_programs <= 64:
return 64, 8, 4
if base_programs <= 128:
return 32, 4, 4
if base_programs <= 256:
return 128, 4, 4
if base_programs <= 512:
return 64, 4, 4
return 32, 2, 1
# bf16 K/V cache.
if base_programs > 2048:
return 64, 2, 2
if base_programs <= 24:
return 64, 64, 8
if base_programs <= 64:
return 64, 8, 8
if base_programs <= 128:
return 32, 8, 8
if base_programs <= 256:
return 32, 8, 1
if base_programs <= 512:
return 64, 4, 2
return 32, 4, 1
def _select_config(
num_rows: int, num_kv_heads: int, use_prefill_config: bool, num_columns: int
num_rows: int,
num_kv_heads: int,
use_prefill_config: bool,
num_columns: int,
is_fp8: bool = False,
) -> tuple[int, int, int, int]:
"""Select (block_n, num_warps, num_tiles, num_splits) for the kernel.
Tuned on GB300 for the Qwen3.8-Flash-Next TP1/TP2/TP4 shapes, keyed on
base_programs = num_rows * num_kv_heads. The bp > 2048 region splits on
use_prefill_config (capture-stable: at FULL-graph capture max_query_len is the
uniform decode/verify length).
Keyed on base_programs = num_rows * num_kv_heads. The bp > 2048 region splits
on use_prefill_config (capture-stable: at FULL-graph capture max_query_len is
the uniform decode/verify length). This default table was tuned on GB300;
sm_120 (RTX PRO 6000 Blackwell) dispatches to _select_sm120_config instead.
"""
base_programs = num_rows * num_kv_heads
if base_programs > 2048:
if _is_sm120():
BLOCK_N, target_splits, num_warps = _select_sm120_config(
base_programs, use_prefill_config, is_fp8
)
elif base_programs > 2048:
BLOCK_N, target_splits, num_warps = (
(32, 1, 1) if use_prefill_config else (64, 1, 2)
)
@@ -483,10 +561,17 @@ def qsa_sparse_paged_attention(
token_to_req: torch.Tensor,
use_prefill_config: bool,
out: torch.Tensor | None = None,
k_scale: float | None = None,
v_scale: float | None = None,
*,
output_gate: torch.Tensor,
) -> torch.Tensor:
"""Run sparse GQA directly over paged BF16 K/V caches.
"""Run sparse GQA directly over paged BF16 or FP8-e4m3 K/V caches.
With fp8 caches, k_scale/v_scale are the layer's per-tensor dequant scales
as host floats (e.g. layer._k_scale_float/_v_scale_float): k_scale is
pre-multiplied into the softmax scale and v_scale becomes the kernel's
output scale, so the kernel needs no device scale buffers.
logical_indices is the PACKED selection buffer: [rows, selection_width + 1]
with the trailing column holding each row's valid-entry count (written by
@@ -510,7 +595,19 @@ def qsa_sparse_paged_attention(
raise ValueError("QSA sparse attention requires valid grouped-query heads")
head_dim = q.shape[2]
assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0
assert q.dtype == k_cache.dtype == v_cache.dtype == torch.bfloat16
assert q.dtype == torch.bfloat16
assert k_cache.dtype == v_cache.dtype
is_fp8 = k_cache.dtype == torch.float8_e4m3fn
if is_fp8:
assert k_scale is not None and v_scale is not None
# Host pre-multiply: fold the K dequant scale into the attention scale
# and pass V's dequant scale as the kernel's output scale.
softmax_scale = (head_dim**-0.5) * float(k_scale)
output_scale = float(v_scale)
else:
assert k_cache.dtype == torch.bfloat16
softmax_scale = head_dim**-0.5
output_scale = 1.0
assert logical_indices.dtype == block_table.dtype == torch.int32
assert token_to_req.dtype == torch.int32
assert q.device == k_cache.device == v_cache.device
@@ -535,7 +632,7 @@ def qsa_sparse_paged_attention(
block_m = triton.next_power_of_2(group_size)
selection_width = logical_indices.shape[1] - 1 # trailing column is the count
block_n, partial_warps, num_tiles, num_splits = _select_config(
q.shape[0], k_cache.shape[2], use_prefill_config, selection_width
q.shape[0], k_cache.shape[2], use_prefill_config, selection_width, is_fp8
)
# Split=1 writes output directly and compiles out all workspace accesses.
@@ -565,6 +662,8 @@ def qsa_sparse_paged_attention(
partial_output,
partial_lse,
out,
softmax_scale,
output_scale,
output_gate_view,
q.stride(0),
q.stride(1),
@@ -593,6 +692,7 @@ def qsa_sparse_paged_attention(
NUM_TILES=num_tiles,
BLOCK_M=block_m,
BLOCK_N=block_n,
IS_FP8=is_fp8,
num_warps=partial_warps,
num_stages=2,
)
@@ -630,13 +730,18 @@ def warmup_qsa_sparse_paged_attention(
head_dim = kv_cache.shape[-1] // 2
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_dim, dim=-1)
# An fp8 cache is allocated as uint8 and viewed as e4m3 at attention time.
is_fp8 = kv_cache.dtype == torch.uint8
cache_dtype = torch.float8_e4m3fn if is_fp8 else key_cache.dtype
num_kv_heads = key_cache.shape[2]
group_size = num_query_heads // num_kv_heads
block_m = triton.next_power_of_2(group_size)
# Every config the dispatch can pick for this group size.
profiles = {
_select_config(num_rows, num_kv_heads, use_prefill_config, selection_width)
_select_config(
num_rows, num_kv_heads, use_prefill_config, selection_width, is_fp8
)
for num_rows in range(1, 8193)
for use_prefill_config in (False, True)
}
@@ -650,12 +755,12 @@ def warmup_qsa_sparse_paged_attention(
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
)
k_cache_ptr = TritonWarmupTensor(
key_cache.dtype,
cache_dtype,
shape=tuple(key_cache.shape),
strides=tuple(key_cache.stride()),
)
v_cache_ptr = TritonWarmupTensor(
value_cache.dtype,
cache_dtype,
shape=tuple(value_cache.shape),
strides=tuple(value_cache.stride()),
)
@@ -701,6 +806,8 @@ def warmup_qsa_sparse_paged_attention(
partial_output_ptr,
partial_lse_ptr,
output_ptr,
1.0,
1.0,
output_gate_ptr,
row_stride,
head_stride,
@@ -729,6 +836,7 @@ def warmup_qsa_sparse_paged_attention(
NUM_TILES=num_tiles,
BLOCK_M=block_m,
BLOCK_N=block_n,
IS_FP8=is_fp8,
num_warps=warps,
num_stages=2,
grid=(num_rows, num_kv_heads, num_splits),
+106 -11
View File
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.transformers_utils.configs.qwen4_exp import (
Qwen4ExpTextConfig,
)
@@ -64,7 +65,38 @@ class Qwen4ExpQSAFlashAttentionBackend(FlashAttentionBackend):
"""FullAttentionSpec backend used by the merged QSA owner."""
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
# fp8/fp8_e4m3: e4m3 bytes in a uint8 cache, written by reshape_and_cache
# with the layer's per-tensor scales and dequantized on load inside the QSA
# Triton kernel. flash-attn never runs over this cache, so its fp8 probe
# does not apply (see supports_kv_cache_dtype and the impl constructor).
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"auto",
"bfloat16",
"fp8",
"fp8_e4m3",
]
@classmethod
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
return kv_cache_dtype is None or kv_cache_dtype in cls.supported_kv_cache_dtypes
@classmethod
def supports_combination(
cls,
head_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType | None,
block_size: int | None,
use_mla: bool,
has_sink: bool,
use_sparse: bool,
use_mm_prefix: bool,
device_capability: DeviceCapability,
) -> str | None:
# QSA dequantizes the fp8 KV in its own Triton kernel and never runs
# flash-attn over the quantized cache, so the parent's fp8-KV rejection
# does not apply and every combination it is handed is accepted here.
return None
@staticmethod
def get_name() -> str:
@@ -98,16 +130,53 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
supports_dcp: bool = False
supports_pcp: bool = False
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int,
alibi_slopes: list[float] | None,
sliding_window: int | None,
kv_cache_dtype: str,
logits_soft_cap: float | None = None,
attn_type: AttentionType = AttentionType.DECODER,
kv_sharing_target_layer_name: str | None = None,
sinks: torch.Tensor | None = None,
) -> None:
# The parent constructor probes flash-attn for quantized-KV support and
# raises where it is unavailable (sm120), but QSA dequantizes fp8 inside
# its own Triton kernel and never runs flash-attn over the cache. Hand
# the parent "auto" for that probe and restore the real dtype afterwards:
# the parent only uses it there, and do_kv_cache_update reads the
# attribute at call time.
real_kv_cache_dtype = kv_cache_dtype
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
kv_cache_dtype = "auto"
super().__init__(
num_heads,
head_size,
scale,
num_kv_heads,
alibi_slopes,
sliding_window,
kv_cache_dtype,
logits_soft_cap,
attn_type,
kv_sharing_target_layer_name,
sinks,
)
self.kv_cache_dtype = real_kv_cache_dtype
if not is_flash_attn_varlen_func_available():
raise NotImplementedError("Qwen4Exp QSA requires FlashAttention")
if self.dcp_world_size != 1:
raise NotImplementedError(
"Qwen4Exp QSA does not support decode context parallelism"
)
if self.kv_cache_dtype not in ("auto", "bfloat16"):
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
if self.kv_cache_dtype not in ("auto", "bfloat16", "fp8", "fp8_e4m3"):
raise NotImplementedError(
"Qwen4Exp QSA requires a BF16 or FP8-e4m3 main KV cache"
)
self.supports_quant_query_input = False
def forward_qsa(
@@ -144,8 +213,23 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
logical_indices = topk_buffer[:num_tokens]
token_to_req = token_to_req[:num_tokens]
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V")
k_scale = v_scale = None
if self.kv_cache_dtype in ("fp8", "fp8_e4m3"):
# The cache is allocated as uint8; reinterpret the e4m3 bytes
# (same itemsize, so shape and strides are preserved).
key_cache = key_cache.view(torch.float8_e4m3fn)
value_cache = value_cache.view(torch.float8_e4m3fn)
# Host-side per-tensor dequant scales (Python floats), as used by
# other host-scale backends; folded into the kernel's scales.
k_scale = layer._k_scale_float
v_scale = layer._v_scale_float
if query.dtype != torch.bfloat16 or key_cache.dtype not in (
torch.bfloat16,
torch.float8_e4m3fn,
):
raise NotImplementedError(
"Qwen4Exp QSA requires BF16 Q and BF16 or FP8-e4m3 K/V"
)
from .ops.qsa import qsa_sparse_paged_attention
@@ -158,6 +242,8 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
token_to_req,
use_prefill_config,
output[:num_tokens],
k_scale=k_scale,
v_scale=v_scale,
output_gate=output_gate[:num_tokens],
)
return output
@@ -185,8 +271,15 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
raise ValueError("Qwen4Exp QSA requires a paged KV cache")
if model_config.dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA currently requires BF16")
if cache_config.cache_dtype not in ("auto", "bfloat16"):
raise NotImplementedError("Qwen4Exp QSA requires a BF16 main KV cache")
if cache_config.cache_dtype not in (
"auto",
"bfloat16",
"fp8",
"fp8_e4m3",
):
raise NotImplementedError(
"Qwen4Exp QSA requires a BF16 or FP8-e4m3 main KV cache"
)
if getattr(quant_config, "kv_cache_scheme", None) is not None:
raise NotImplementedError("Qwen4Exp QSA does not support KV quantization")
parallel_config = vllm_config.parallel_config
@@ -285,8 +378,10 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
self.kv_cache_dtype, model_config
)
if self.kv_cache_torch_dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA requires BF16 cache storage")
if self.kv_cache_torch_dtype not in (torch.bfloat16, torch.uint8):
raise NotImplementedError(
"Qwen4Exp QSA requires BF16 or FP8-e4m3 (uint8) cache storage"
)
self.kv_sharing_target_layer_name = None
self.kv_cache = torch.tensor([])
set_default_quant_scales(self, register_buffer=True)