[Qwen3.8-Flash-Next] Support FP8 indexer cache for QSA (#54890)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-09-07 12:15:45 +00:00
committed by GitHub
parent 58ad1f3b89
commit 94e26dd3dd
5 changed files with 142 additions and 24 deletions
+31 -6
View File
@@ -49,8 +49,21 @@ def _make_block_table(block_counts):
return table, num_blocks
def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> None:
# e4m3 is sign-magnitude, so within a sign the uint8 code order matches the
# value order and one ulp is one code step. The two paths' intermediates
# differ in the pooling accumulation order, which at denormal magnitudes
# (absolute grid step 2^-9) shows up as up to 2 code steps.
code_diff = (
actual.view(torch.uint8).int() - expected.view(torch.uint8).int()
).abs()
abs_diff = (actual.float() - expected.float()).abs()
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize(
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
[
@@ -74,6 +87,7 @@ def _make_block_table(block_counts):
],
)
def test_qsa_fused_pre_indexer_matches_unfused(
indexer_dtype,
mrope,
is_2d_positions,
cache_rope_positions,
@@ -215,7 +229,7 @@ def test_qsa_fused_pre_indexer_matches_unfused(
fused_compressed_storage = torch.zeros(
num_compressed_blocks,
compressed_page_elements + 16,
dtype=torch.bfloat16,
dtype=indexer_dtype,
device=device,
)
fused_compressed = torch.as_strided(
@@ -231,7 +245,7 @@ def test_qsa_fused_pre_indexer_matches_unfused(
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
fused_query = torch.empty(num_tokens, HQ, D, dtype=torch.bfloat16, device=device)
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
qsa_pre_indexer(
projected_qk[:, : HQ * D],
projected_qk[:, HQ * D :],
@@ -290,8 +304,19 @@ def test_qsa_fused_pre_indexer_matches_unfused(
if rope_positions is not None:
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
if indexer_dtype == torch.float8_e4m3fn:
# Both paths round the same ~bf16 intermediates to e4m3. Bitwise
# equality (the dsv4 indexer test's bar) does not hold here: the 1D
# reference rope (forward_cuda) differs from _norm_rope by up to 1
# bf16 ulp, and even the MRoPE pair flips a code occasionally.
unfused_query = unfused_query.to(indexer_dtype)
assert_fp8_within_one_ulp(fused_query, unfused_query)
else:
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
torch.testing.assert_close(
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
)
if indexer_dtype == torch.float8_e4m3fn:
assert_fp8_within_one_ulp(fused_compressed, unfused_compressed)
else:
torch.testing.assert_close(
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
)
+61 -6
View File
@@ -48,6 +48,7 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
index_n_heads=1,
index_kv_heads=1,
index_head_dim=1,
indexer_dtype=torch.bfloat16,
raw_key_cache=SimpleNamespace(
kv_cache=torch.empty(0),
rope_position_cache=None,
@@ -484,6 +485,7 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
use_fused_pre_indexer=False,
index_n_heads=1,
index_head_dim=64,
indexer_dtype=torch.bfloat16,
q_layernorm=norm,
k_layernorm=norm,
rotary_emb=rope,
@@ -666,13 +668,16 @@ def test_qsa_fused_metadata_matches_pytorch_for_large_padded_prefill() -> None:
(4, 33),
],
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
def test_qsa_decode_selection_correctness(
decode_query_len: int, num_requests: int
decode_query_len: int, num_requests: int, dtype: torch.dtype
) -> None:
torch.manual_seed(1)
heads, head_dim = 4, 128
rows = num_requests * decode_query_len
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
dtype
)
page_size, pages_per_request, max_sequence_length = (
(16, 40, 2560) if num_requests > 32 else (4, 20, 320)
)
@@ -684,7 +689,7 @@ def test_qsa_decode_selection_correctness(
head_dim,
device="cuda",
dtype=torch.bfloat16,
)
).to(dtype)
page_table = torch.randperm(num_pages, device="cuda", dtype=torch.int32).reshape(
num_requests, pages_per_request
)
@@ -736,14 +741,39 @@ def test_qsa_decode_selection_correctness(
compress_ratio,
)
if dtype == torch.float8_e4m3fn:
# fp8 logits tie at the top-k boundary more often than bf16, so index
# identity is not stable; compare the selected value multisets.
# SM90 wgmma accumulates fp8 in reduced precision (~3e-4 abs
# observed); SM100 tcgen05 is exact fp32.
rtol = atol = 1e-3 if current_platform.is_device_capability(90) else None
logits = _qsa_mqa_paged_reference(
q, cache, page_table, token_to_req, visible_blocks
)
for row in range(rows):
selected = actual[row][actual[row] >= 0]
wanted = expected[row][expected[row] >= 0]
assert selected.numel() == wanted.numel()
torch.testing.assert_close(
logits[row, selected.long()].sort().values,
logits[row, wanted.long()].sort().values,
rtol=rtol,
atol=atol,
)
return
torch.testing.assert_close(actual.sort().values, expected.sort().values)
@requires_qsa_kernels
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize("seq_len_slack", [0, 1792])
@pytest.mark.parametrize("force_chunk", [False, True])
def test_qsa_prefill_selection_correctness(
monkeypatch: pytest.MonkeyPatch, seq_len_slack: int, force_chunk: bool
monkeypatch: pytest.MonkeyPatch,
seq_len_slack: int,
force_chunk: bool,
dtype: torch.dtype,
) -> None:
# page_size=24 (does not divide the 64-aligned clipped width) and an
# oversized page table, so the clipped logits width comes from
@@ -755,8 +785,12 @@ def test_qsa_prefill_selection_correctness(
torch.manual_seed(2)
query_lens = [3, 33]
rows, heads, head_dim = sum(query_lens), 4, 128
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
cache = torch.randn(128, 24, 1, head_dim, device="cuda", dtype=torch.bfloat16)
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
dtype
)
cache = torch.randn(128, 24, 1, head_dim, device="cuda", dtype=torch.bfloat16).to(
dtype
)
page_table = torch.randperm(128, device="cuda", dtype=torch.int32).reshape(2, 64)
token_to_req = torch.repeat_interleave(
torch.arange(2, device="cuda", dtype=torch.int32),
@@ -804,6 +838,27 @@ def test_qsa_prefill_selection_correctness(
compress_ratio,
)
if dtype == torch.float8_e4m3fn:
# fp8 logits tie at the top-k boundary more often than bf16, so index
# identity is not stable; compare the selected value multisets.
# SM90 wgmma accumulates fp8 in reduced precision (~3e-4 abs
# observed); SM100 tcgen05 is exact fp32.
rtol = atol = 1e-3 if current_platform.is_device_capability(90) else None
logits = _qsa_mqa_paged_reference(
q, cache, page_table, token_to_req, visible_blocks
)
for row in range(rows):
selected = actual[row][actual[row] >= 0]
wanted = expected[row][expected[row] >= 0]
assert selected.numel() == wanted.numel()
torch.testing.assert_close(
logits[row, selected.long()].sort().values,
logits[row, wanted.long()].sort().values,
rtol=rtol,
atol=atol,
)
return
torch.testing.assert_close(actual.sort().values, expected.sort().values)