[Qwen3.8-Flash-Next] Support FP8 indexer cache for QSA (#54890)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -49,8 +49,21 @@ def _make_block_table(block_counts):
|
||||
return table, num_blocks
|
||||
|
||||
|
||||
def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
# e4m3 is sign-magnitude, so within a sign the uint8 code order matches the
|
||||
# value order and one ulp is one code step. The two paths' intermediates
|
||||
# differ in the pooling accumulation order, which at denormal magnitudes
|
||||
# (absolute grid step 2^-9) shows up as up to 2 code steps.
|
||||
code_diff = (
|
||||
actual.view(torch.uint8).int() - expected.view(torch.uint8).int()
|
||||
).abs()
|
||||
abs_diff = (actual.float() - expected.float()).abs()
|
||||
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize(
|
||||
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
|
||||
[
|
||||
@@ -74,6 +87,7 @@ def _make_block_table(block_counts):
|
||||
],
|
||||
)
|
||||
def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
indexer_dtype,
|
||||
mrope,
|
||||
is_2d_positions,
|
||||
cache_rope_positions,
|
||||
@@ -215,7 +229,7 @@ def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
fused_compressed_storage = torch.zeros(
|
||||
num_compressed_blocks,
|
||||
compressed_page_elements + 16,
|
||||
dtype=torch.bfloat16,
|
||||
dtype=indexer_dtype,
|
||||
device=device,
|
||||
)
|
||||
fused_compressed = torch.as_strided(
|
||||
@@ -231,7 +245,7 @@ def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
fused_query = torch.empty(num_tokens, HQ, D, dtype=torch.bfloat16, device=device)
|
||||
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
|
||||
qsa_pre_indexer(
|
||||
projected_qk[:, : HQ * D],
|
||||
projected_qk[:, HQ * D :],
|
||||
@@ -290,8 +304,19 @@ def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
if rope_positions is not None:
|
||||
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
|
||||
|
||||
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
|
||||
if indexer_dtype == torch.float8_e4m3fn:
|
||||
# Both paths round the same ~bf16 intermediates to e4m3. Bitwise
|
||||
# equality (the dsv4 indexer test's bar) does not hold here: the 1D
|
||||
# reference rope (forward_cuda) differs from _norm_rope by up to 1
|
||||
# bf16 ulp, and even the MRoPE pair flips a code occasionally.
|
||||
unfused_query = unfused_query.to(indexer_dtype)
|
||||
assert_fp8_within_one_ulp(fused_query, unfused_query)
|
||||
else:
|
||||
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
|
||||
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
|
||||
torch.testing.assert_close(
|
||||
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
|
||||
)
|
||||
if indexer_dtype == torch.float8_e4m3fn:
|
||||
assert_fp8_within_one_ulp(fused_compressed, unfused_compressed)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
|
||||
)
|
||||
|
||||
@@ -48,6 +48,7 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
index_n_heads=1,
|
||||
index_kv_heads=1,
|
||||
index_head_dim=1,
|
||||
indexer_dtype=torch.bfloat16,
|
||||
raw_key_cache=SimpleNamespace(
|
||||
kv_cache=torch.empty(0),
|
||||
rope_position_cache=None,
|
||||
@@ -484,6 +485,7 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
|
||||
use_fused_pre_indexer=False,
|
||||
index_n_heads=1,
|
||||
index_head_dim=64,
|
||||
indexer_dtype=torch.bfloat16,
|
||||
q_layernorm=norm,
|
||||
k_layernorm=norm,
|
||||
rotary_emb=rope,
|
||||
@@ -666,13 +668,16 @@ def test_qsa_fused_metadata_matches_pytorch_for_large_padded_prefill() -> None:
|
||||
(4, 33),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
def test_qsa_decode_selection_correctness(
|
||||
decode_query_len: int, num_requests: int
|
||||
decode_query_len: int, num_requests: int, dtype: torch.dtype
|
||||
) -> None:
|
||||
torch.manual_seed(1)
|
||||
heads, head_dim = 4, 128
|
||||
rows = num_requests * decode_query_len
|
||||
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
|
||||
dtype
|
||||
)
|
||||
page_size, pages_per_request, max_sequence_length = (
|
||||
(16, 40, 2560) if num_requests > 32 else (4, 20, 320)
|
||||
)
|
||||
@@ -684,7 +689,7 @@ def test_qsa_decode_selection_correctness(
|
||||
head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
).to(dtype)
|
||||
page_table = torch.randperm(num_pages, device="cuda", dtype=torch.int32).reshape(
|
||||
num_requests, pages_per_request
|
||||
)
|
||||
@@ -736,14 +741,39 @@ def test_qsa_decode_selection_correctness(
|
||||
compress_ratio,
|
||||
)
|
||||
|
||||
if dtype == torch.float8_e4m3fn:
|
||||
# fp8 logits tie at the top-k boundary more often than bf16, so index
|
||||
# identity is not stable; compare the selected value multisets.
|
||||
# SM90 wgmma accumulates fp8 in reduced precision (~3e-4 abs
|
||||
# observed); SM100 tcgen05 is exact fp32.
|
||||
rtol = atol = 1e-3 if current_platform.is_device_capability(90) else None
|
||||
logits = _qsa_mqa_paged_reference(
|
||||
q, cache, page_table, token_to_req, visible_blocks
|
||||
)
|
||||
for row in range(rows):
|
||||
selected = actual[row][actual[row] >= 0]
|
||||
wanted = expected[row][expected[row] >= 0]
|
||||
assert selected.numel() == wanted.numel()
|
||||
torch.testing.assert_close(
|
||||
logits[row, selected.long()].sort().values,
|
||||
logits[row, wanted.long()].sort().values,
|
||||
rtol=rtol,
|
||||
atol=atol,
|
||||
)
|
||||
return
|
||||
|
||||
torch.testing.assert_close(actual.sort().values, expected.sort().values)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize("seq_len_slack", [0, 1792])
|
||||
@pytest.mark.parametrize("force_chunk", [False, True])
|
||||
def test_qsa_prefill_selection_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch, seq_len_slack: int, force_chunk: bool
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
seq_len_slack: int,
|
||||
force_chunk: bool,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
# page_size=24 (does not divide the 64-aligned clipped width) and an
|
||||
# oversized page table, so the clipped logits width comes from
|
||||
@@ -755,8 +785,12 @@ def test_qsa_prefill_selection_correctness(
|
||||
torch.manual_seed(2)
|
||||
query_lens = [3, 33]
|
||||
rows, heads, head_dim = sum(query_lens), 4, 128
|
||||
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.randn(128, 24, 1, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
|
||||
dtype
|
||||
)
|
||||
cache = torch.randn(128, 24, 1, head_dim, device="cuda", dtype=torch.bfloat16).to(
|
||||
dtype
|
||||
)
|
||||
page_table = torch.randperm(128, device="cuda", dtype=torch.int32).reshape(2, 64)
|
||||
token_to_req = torch.repeat_interleave(
|
||||
torch.arange(2, device="cuda", dtype=torch.int32),
|
||||
@@ -804,6 +838,27 @@ def test_qsa_prefill_selection_correctness(
|
||||
compress_ratio,
|
||||
)
|
||||
|
||||
if dtype == torch.float8_e4m3fn:
|
||||
# fp8 logits tie at the top-k boundary more often than bf16, so index
|
||||
# identity is not stable; compare the selected value multisets.
|
||||
# SM90 wgmma accumulates fp8 in reduced precision (~3e-4 abs
|
||||
# observed); SM100 tcgen05 is exact fp32.
|
||||
rtol = atol = 1e-3 if current_platform.is_device_capability(90) else None
|
||||
logits = _qsa_mqa_paged_reference(
|
||||
q, cache, page_table, token_to_req, visible_blocks
|
||||
)
|
||||
for row in range(rows):
|
||||
selected = actual[row][actual[row] >= 0]
|
||||
wanted = expected[row][expected[row] >= 0]
|
||||
assert selected.numel() == wanted.numel()
|
||||
torch.testing.assert_close(
|
||||
logits[row, selected.long()].sort().values,
|
||||
logits[row, wanted.long()].sort().values,
|
||||
rtol=rtol,
|
||||
atol=atol,
|
||||
)
|
||||
return
|
||||
|
||||
torch.testing.assert_close(actual.sort().values, expected.sort().values)
|
||||
|
||||
|
||||
|
||||
@@ -725,10 +725,16 @@ class QSAMetadataBuilder(AttentionMetadataBuilder[QSAForwardMetadata]):
|
||||
|
||||
|
||||
class QSAStateBackend(AttentionBackend):
|
||||
"""Key-only dummy backend for out-of-band BF16 QSA side-cache operations."""
|
||||
"""Key-only dummy backend for out-of-band QSA side-cache operations."""
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"]
|
||||
# fp8 entries allow the optional e4m3 compressed indexer cache.
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"bfloat16",
|
||||
"fp8",
|
||||
"fp8_e4m3",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
@@ -790,8 +796,12 @@ class _QSAStateCache(nn.Module, AttentionLayerBase):
|
||||
"""Adapt the unified [B, H, N, C] view to QSA's [B, N, H, C]."""
|
||||
if kv_cache.ndim != 4 or kv_cache.shape[1] != 1:
|
||||
raise ValueError("QSA state cache must be [blocks, 1, states, width]")
|
||||
if kv_cache.dtype != torch.bfloat16 or kv_cache.shape[3] != self.head_size:
|
||||
raise ValueError("QSA state cache does not match its packed BF16 spec")
|
||||
if kv_cache.dtype != self.dtype or kv_cache.shape[3] != self.head_size:
|
||||
raise ValueError(
|
||||
f"QSA state cache does not match its spec "
|
||||
f"(dtype {kv_cache.dtype} != {self.dtype} or width "
|
||||
f"{kv_cache.shape[3]} != {self.head_size})"
|
||||
)
|
||||
super().bind_kv_cache(kv_cache.transpose(1, 2))
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
@@ -850,7 +860,7 @@ class QSAKeyStateCache(_QSAStateCache):
|
||||
|
||||
|
||||
class QSACompressedKeyCache(_QSAStateCache):
|
||||
"""Normalized, group-first-RoPE BF16 key at one row per complete group."""
|
||||
"""Normed, group-first-RoPE key at one row per complete group."""
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
del vllm_config
|
||||
|
||||
@@ -147,6 +147,21 @@ class QSAIndexer(nn.Module):
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
cache_prefix = f"{prefix}." if prefix else ""
|
||||
# Plain e4m3 without scales: Q and the compressed K are RMSNormed
|
||||
# before quantization, and the logits kernels dot fp8 x fp8 directly.
|
||||
self.indexer_kv_dtype = vllm_config.attention_config.resolve_indexer_kv_dtype(
|
||||
"bf16"
|
||||
)
|
||||
if self.indexer_kv_dtype == "fp8":
|
||||
indexer_dtype = torch.float8_e4m3fn
|
||||
elif self.indexer_kv_dtype == "bf16":
|
||||
indexer_dtype = torch.bfloat16
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"indexer_kv_dtype={self.indexer_kv_dtype!r} is not supported "
|
||||
"by the Qwen4Exp QSA indexer (only 'bf16' or 'fp8')."
|
||||
)
|
||||
self.indexer_dtype = indexer_dtype
|
||||
self.raw_key_cache = QSAKeyStateCache(
|
||||
head_size=self.index_head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
@@ -158,7 +173,7 @@ class QSAIndexer(nn.Module):
|
||||
)
|
||||
self.compressed_key_cache = QSACompressedKeyCache(
|
||||
head_size=self.index_head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
dtype=indexer_dtype,
|
||||
compress_ratio=self.compress_ratio,
|
||||
prefix=f"{cache_prefix}compressed_key_cache",
|
||||
cache_config=cache_config,
|
||||
@@ -276,6 +291,7 @@ class QSAIndexer(nn.Module):
|
||||
num_tokens,
|
||||
self.index_n_heads,
|
||||
self.index_head_dim,
|
||||
dtype=self.indexer_dtype,
|
||||
)
|
||||
qsa_pre_indexer(
|
||||
projected_q,
|
||||
@@ -313,6 +329,7 @@ class QSAIndexer(nn.Module):
|
||||
self.q_layernorm.variance_epsilon,
|
||||
).reshape_as(q)
|
||||
q = apply_qsa_rope(self.rotary_emb, positions, q)
|
||||
q = q.to(self.indexer_dtype)
|
||||
|
||||
raw_key_cache = raw_key_state_cache.key_cache
|
||||
rope_position_cache = raw_key_state_cache.rope_position_cache
|
||||
|
||||
@@ -200,6 +200,9 @@ def _qsa_mqa_paged_prefill_kernel(
|
||||
other=0.0,
|
||||
eviction_policy="evict_first",
|
||||
)
|
||||
# With an fp8 cache, SM90 lowers this dot to reduced-precision wgmma
|
||||
# accumulation; SM100 tcgen05 accumulates in fp32. The SM90 error
|
||||
# (~1e-4 relative) is far below the e4m3 quantization noise floor.
|
||||
scores = tl.dot(keys, query, out_dtype=tl.float32)
|
||||
scores = tl.reshape(scores, (BLOCK_N, TILE_R, NUM_HEADS_PADDED))
|
||||
score = tl.sum(tl.maximum(scores, 0.0), axis=2)
|
||||
@@ -337,7 +340,7 @@ def warmup_qsa_mqa_paged_decode(
|
||||
for decode_query_len, num_requests in profiles:
|
||||
num_rows = decode_query_len * num_requests
|
||||
q_ptr = TritonWarmupTensor(
|
||||
torch.bfloat16,
|
||||
k_cache.dtype,
|
||||
shape=(num_rows, num_heads, head_dim),
|
||||
)
|
||||
logits_ptr = TritonWarmupTensor(
|
||||
@@ -365,7 +368,8 @@ def warmup_qsa_mqa_paged_decode(
|
||||
BLOCK_N=_DECODE_BLOCK_N,
|
||||
TILES_PER_PROG=tiles_per_program,
|
||||
STAGES=2,
|
||||
num_warps=2,
|
||||
# tuned on GB300
|
||||
num_warps=1 if k_cache.dtype == torch.float8_e4m3fn else 2,
|
||||
grid=(
|
||||
num_requests,
|
||||
triton.cdiv(columns, _DECODE_BLOCK_N * tiles_per_program),
|
||||
@@ -393,7 +397,11 @@ def _prefill_logits(
|
||||
logits = torch.empty(
|
||||
(num_queries, logits_width), dtype=torch.float32, device=q.device
|
||||
)
|
||||
TILE_R = 64
|
||||
# tuned on GB300
|
||||
if k_cache.dtype == torch.float8_e4m3fn:
|
||||
TILE_R, STAGES, num_warps = 32, 2, 8
|
||||
else:
|
||||
TILE_R, STAGES, num_warps = 64, 2, 4
|
||||
BLOCK_N = 64
|
||||
K_TILES = 16
|
||||
grid = (
|
||||
@@ -421,8 +429,8 @@ def _prefill_logits(
|
||||
TILE_R=TILE_R,
|
||||
BLOCK_N=BLOCK_N,
|
||||
K_TILES=K_TILES,
|
||||
STAGES=2,
|
||||
num_warps=4,
|
||||
STAGES=STAGES,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
return logits
|
||||
|
||||
@@ -521,6 +529,7 @@ def qsa_select_paged_decode(
|
||||
assert token_topk % compress_ratio == 0
|
||||
assert block_indices.shape == (q.shape[0], token_topk // compress_ratio)
|
||||
assert decode_query_len > 0 and q.shape[0] % decode_query_len == 0
|
||||
assert q.dtype == k_cache.dtype, "Q and the compressed K cache must match"
|
||||
num_requests = q.shape[0] // decode_query_len
|
||||
assert page_table.shape[0] == num_requests
|
||||
assert visible_blocks.shape == (q.shape[0],)
|
||||
@@ -550,7 +559,8 @@ def qsa_select_paged_decode(
|
||||
BLOCK_N=_DECODE_BLOCK_N,
|
||||
TILES_PER_PROG=tiles_per_program,
|
||||
STAGES=2,
|
||||
num_warps=2,
|
||||
# tuned on GB300
|
||||
num_warps=1 if k_cache.dtype == torch.float8_e4m3fn else 2,
|
||||
)
|
||||
_topk(
|
||||
logits,
|
||||
@@ -593,6 +603,7 @@ def qsa_select_paged_prefill(
|
||||
|
||||
assert token_topk % compress_ratio == 0
|
||||
assert block_indices.shape == (q.shape[0], token_topk // compress_ratio)
|
||||
assert q.dtype == k_cache.dtype, "Q and the compressed K cache must match"
|
||||
rows = q.shape[0]
|
||||
# No row scores beyond cdiv(max_seq_len, compress_ratio) compressed
|
||||
# columns. Round up to 64 to keep the logits row stride
|
||||
|
||||
Reference in New Issue
Block a user