diff --git a/tests/models/qwen4_exp/test_qsa_pre_indexer.py b/tests/models/qwen4_exp/test_qsa_pre_indexer.py index 991c73af00..8d4df3dbc9 100644 --- a/tests/models/qwen4_exp/test_qsa_pre_indexer.py +++ b/tests/models/qwen4_exp/test_qsa_pre_indexer.py @@ -49,8 +49,21 @@ def _make_block_table(block_counts): return table, num_blocks +def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> None: + # e4m3 is sign-magnitude, so within a sign the uint8 code order matches the + # value order and one ulp is one code step. The two paths' intermediates + # differ in the pooling accumulation order, which at denormal magnitudes + # (absolute grid step 2^-9) shows up as up to 2 code steps. + code_diff = ( + actual.view(torch.uint8).int() - expected.view(torch.uint8).int() + ).abs() + abs_diff = (actual.float() - expected.float()).abs() + assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all()) + + @requires_qsa_kernels @pytest.mark.usefixtures("default_vllm_config") +@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn]) @pytest.mark.parametrize( "mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens", [ @@ -74,6 +87,7 @@ def _make_block_table(block_counts): ], ) def test_qsa_fused_pre_indexer_matches_unfused( + indexer_dtype, mrope, is_2d_positions, cache_rope_positions, @@ -215,7 +229,7 @@ def test_qsa_fused_pre_indexer_matches_unfused( fused_compressed_storage = torch.zeros( num_compressed_blocks, compressed_page_elements + 16, - dtype=torch.bfloat16, + dtype=indexer_dtype, device=device, ) fused_compressed = torch.as_strided( @@ -231,7 +245,7 @@ def test_qsa_fused_pre_indexer_matches_unfused( q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2 k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2 - fused_query = torch.empty(num_tokens, HQ, D, dtype=torch.bfloat16, device=device) + fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device) qsa_pre_indexer( projected_qk[:, : HQ * D], projected_qk[:, HQ * D :], @@ -290,8 +304,19 @@ def test_qsa_fused_pre_indexer_matches_unfused( if rope_positions is not None: qsa_store_cache_rows(rope_positions, raw_slots, position_rows) - torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL) + if indexer_dtype == torch.float8_e4m3fn: + # Both paths round the same ~bf16 intermediates to e4m3. Bitwise + # equality (the dsv4 indexer test's bar) does not hold here: the 1D + # reference rope (forward_cuda) differs from _norm_rope by up to 1 + # bf16 ulp, and even the MRoPE pair flips a code occasionally. + unfused_query = unfused_query.to(indexer_dtype) + assert_fp8_within_one_ulp(fused_query, unfused_query) + else: + torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL) assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16)) - torch.testing.assert_close( - fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL - ) + if indexer_dtype == torch.float8_e4m3fn: + assert_fp8_within_one_ulp(fused_compressed, unfused_compressed) + else: + torch.testing.assert_close( + fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL + ) diff --git a/tests/models/qwen4_exp/test_qsa_reference.py b/tests/models/qwen4_exp/test_qsa_reference.py index eb026e2205..9259972ac9 100644 --- a/tests/models/qwen4_exp/test_qsa_reference.py +++ b/tests/models/qwen4_exp/test_qsa_reference.py @@ -48,6 +48,7 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection( index_n_heads=1, index_kv_heads=1, index_head_dim=1, + indexer_dtype=torch.bfloat16, raw_key_cache=SimpleNamespace( kv_cache=torch.empty(0), rope_position_cache=None, @@ -484,6 +485,7 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None: use_fused_pre_indexer=False, index_n_heads=1, index_head_dim=64, + indexer_dtype=torch.bfloat16, q_layernorm=norm, k_layernorm=norm, rotary_emb=rope, @@ -666,13 +668,16 @@ def test_qsa_fused_metadata_matches_pytorch_for_large_padded_prefill() -> None: (4, 33), ], ) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn]) def test_qsa_decode_selection_correctness( - decode_query_len: int, num_requests: int + decode_query_len: int, num_requests: int, dtype: torch.dtype ) -> None: torch.manual_seed(1) heads, head_dim = 4, 128 rows = num_requests * decode_query_len - q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16) + q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16).to( + dtype + ) page_size, pages_per_request, max_sequence_length = ( (16, 40, 2560) if num_requests > 32 else (4, 20, 320) ) @@ -684,7 +689,7 @@ def test_qsa_decode_selection_correctness( head_dim, device="cuda", dtype=torch.bfloat16, - ) + ).to(dtype) page_table = torch.randperm(num_pages, device="cuda", dtype=torch.int32).reshape( num_requests, pages_per_request ) @@ -736,14 +741,39 @@ def test_qsa_decode_selection_correctness( compress_ratio, ) + if dtype == torch.float8_e4m3fn: + # fp8 logits tie at the top-k boundary more often than bf16, so index + # identity is not stable; compare the selected value multisets. + # SM90 wgmma accumulates fp8 in reduced precision (~3e-4 abs + # observed); SM100 tcgen05 is exact fp32. + rtol = atol = 1e-3 if current_platform.is_device_capability(90) else None + logits = _qsa_mqa_paged_reference( + q, cache, page_table, token_to_req, visible_blocks + ) + for row in range(rows): + selected = actual[row][actual[row] >= 0] + wanted = expected[row][expected[row] >= 0] + assert selected.numel() == wanted.numel() + torch.testing.assert_close( + logits[row, selected.long()].sort().values, + logits[row, wanted.long()].sort().values, + rtol=rtol, + atol=atol, + ) + return + torch.testing.assert_close(actual.sort().values, expected.sort().values) @requires_qsa_kernels +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn]) @pytest.mark.parametrize("seq_len_slack", [0, 1792]) @pytest.mark.parametrize("force_chunk", [False, True]) def test_qsa_prefill_selection_correctness( - monkeypatch: pytest.MonkeyPatch, seq_len_slack: int, force_chunk: bool + monkeypatch: pytest.MonkeyPatch, + seq_len_slack: int, + force_chunk: bool, + dtype: torch.dtype, ) -> None: # page_size=24 (does not divide the 64-aligned clipped width) and an # oversized page table, so the clipped logits width comes from @@ -755,8 +785,12 @@ def test_qsa_prefill_selection_correctness( torch.manual_seed(2) query_lens = [3, 33] rows, heads, head_dim = sum(query_lens), 4, 128 - q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16) - cache = torch.randn(128, 24, 1, head_dim, device="cuda", dtype=torch.bfloat16) + q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16).to( + dtype + ) + cache = torch.randn(128, 24, 1, head_dim, device="cuda", dtype=torch.bfloat16).to( + dtype + ) page_table = torch.randperm(128, device="cuda", dtype=torch.int32).reshape(2, 64) token_to_req = torch.repeat_interleave( torch.arange(2, device="cuda", dtype=torch.int32), @@ -804,6 +838,27 @@ def test_qsa_prefill_selection_correctness( compress_ratio, ) + if dtype == torch.float8_e4m3fn: + # fp8 logits tie at the top-k boundary more often than bf16, so index + # identity is not stable; compare the selected value multisets. + # SM90 wgmma accumulates fp8 in reduced precision (~3e-4 abs + # observed); SM100 tcgen05 is exact fp32. + rtol = atol = 1e-3 if current_platform.is_device_capability(90) else None + logits = _qsa_mqa_paged_reference( + q, cache, page_table, token_to_req, visible_blocks + ) + for row in range(rows): + selected = actual[row][actual[row] >= 0] + wanted = expected[row][expected[row] >= 0] + assert selected.numel() == wanted.numel() + torch.testing.assert_close( + logits[row, selected.long()].sort().values, + logits[row, wanted.long()].sort().values, + rtol=rtol, + atol=atol, + ) + return + torch.testing.assert_close(actual.sort().values, expected.sort().values) diff --git a/vllm/models/qwen4_exp/common/qsa_cache.py b/vllm/models/qwen4_exp/common/qsa_cache.py index acddd4f9e6..e806dc0375 100644 --- a/vllm/models/qwen4_exp/common/qsa_cache.py +++ b/vllm/models/qwen4_exp/common/qsa_cache.py @@ -725,10 +725,16 @@ class QSAMetadataBuilder(AttentionMetadataBuilder[QSAForwardMetadata]): class QSAStateBackend(AttentionBackend): - """Key-only dummy backend for out-of-band BF16 QSA side-cache operations.""" + """Key-only dummy backend for out-of-band QSA side-cache operations.""" supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] - supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["auto", "bfloat16"] + # fp8 entries allow the optional e4m3 compressed indexer cache. + supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ + "auto", + "bfloat16", + "fp8", + "fp8_e4m3", + ] @staticmethod def get_name() -> str: @@ -790,8 +796,12 @@ class _QSAStateCache(nn.Module, AttentionLayerBase): """Adapt the unified [B, H, N, C] view to QSA's [B, N, H, C].""" if kv_cache.ndim != 4 or kv_cache.shape[1] != 1: raise ValueError("QSA state cache must be [blocks, 1, states, width]") - if kv_cache.dtype != torch.bfloat16 or kv_cache.shape[3] != self.head_size: - raise ValueError("QSA state cache does not match its packed BF16 spec") + if kv_cache.dtype != self.dtype or kv_cache.shape[3] != self.head_size: + raise ValueError( + f"QSA state cache does not match its spec " + f"(dtype {kv_cache.dtype} != {self.dtype} or width " + f"{kv_cache.shape[3]} != {self.head_size})" + ) super().bind_kv_cache(kv_cache.transpose(1, 2)) def get_attn_backend(self) -> type[AttentionBackend]: @@ -850,7 +860,7 @@ class QSAKeyStateCache(_QSAStateCache): class QSACompressedKeyCache(_QSAStateCache): - """Normalized, group-first-RoPE BF16 key at one row per complete group.""" + """Normed, group-first-RoPE key at one row per complete group.""" def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: del vllm_config diff --git a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py index b14174e295..bbdc5e4feb 100644 --- a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py +++ b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py @@ -147,6 +147,21 @@ class QSAIndexer(nn.Module): cache_config = vllm_config.cache_config cache_prefix = f"{prefix}." if prefix else "" + # Plain e4m3 without scales: Q and the compressed K are RMSNormed + # before quantization, and the logits kernels dot fp8 x fp8 directly. + self.indexer_kv_dtype = vllm_config.attention_config.resolve_indexer_kv_dtype( + "bf16" + ) + if self.indexer_kv_dtype == "fp8": + indexer_dtype = torch.float8_e4m3fn + elif self.indexer_kv_dtype == "bf16": + indexer_dtype = torch.bfloat16 + else: + raise NotImplementedError( + f"indexer_kv_dtype={self.indexer_kv_dtype!r} is not supported " + "by the Qwen4Exp QSA indexer (only 'bf16' or 'fp8')." + ) + self.indexer_dtype = indexer_dtype self.raw_key_cache = QSAKeyStateCache( head_size=self.index_head_dim, dtype=torch.bfloat16, @@ -158,7 +173,7 @@ class QSAIndexer(nn.Module): ) self.compressed_key_cache = QSACompressedKeyCache( head_size=self.index_head_dim, - dtype=torch.bfloat16, + dtype=indexer_dtype, compress_ratio=self.compress_ratio, prefix=f"{cache_prefix}compressed_key_cache", cache_config=cache_config, @@ -276,6 +291,7 @@ class QSAIndexer(nn.Module): num_tokens, self.index_n_heads, self.index_head_dim, + dtype=self.indexer_dtype, ) qsa_pre_indexer( projected_q, @@ -313,6 +329,7 @@ class QSAIndexer(nn.Module): self.q_layernorm.variance_epsilon, ).reshape_as(q) q = apply_qsa_rope(self.rotary_emb, positions, q) + q = q.to(self.indexer_dtype) raw_key_cache = raw_key_state_cache.key_cache rope_position_cache = raw_key_state_cache.rope_position_cache diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py index 7efcc9077b..9ddd65d159 100644 --- a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py @@ -200,6 +200,9 @@ def _qsa_mqa_paged_prefill_kernel( other=0.0, eviction_policy="evict_first", ) + # With an fp8 cache, SM90 lowers this dot to reduced-precision wgmma + # accumulation; SM100 tcgen05 accumulates in fp32. The SM90 error + # (~1e-4 relative) is far below the e4m3 quantization noise floor. scores = tl.dot(keys, query, out_dtype=tl.float32) scores = tl.reshape(scores, (BLOCK_N, TILE_R, NUM_HEADS_PADDED)) score = tl.sum(tl.maximum(scores, 0.0), axis=2) @@ -337,7 +340,7 @@ def warmup_qsa_mqa_paged_decode( for decode_query_len, num_requests in profiles: num_rows = decode_query_len * num_requests q_ptr = TritonWarmupTensor( - torch.bfloat16, + k_cache.dtype, shape=(num_rows, num_heads, head_dim), ) logits_ptr = TritonWarmupTensor( @@ -365,7 +368,8 @@ def warmup_qsa_mqa_paged_decode( BLOCK_N=_DECODE_BLOCK_N, TILES_PER_PROG=tiles_per_program, STAGES=2, - num_warps=2, + # tuned on GB300 + num_warps=1 if k_cache.dtype == torch.float8_e4m3fn else 2, grid=( num_requests, triton.cdiv(columns, _DECODE_BLOCK_N * tiles_per_program), @@ -393,7 +397,11 @@ def _prefill_logits( logits = torch.empty( (num_queries, logits_width), dtype=torch.float32, device=q.device ) - TILE_R = 64 + # tuned on GB300 + if k_cache.dtype == torch.float8_e4m3fn: + TILE_R, STAGES, num_warps = 32, 2, 8 + else: + TILE_R, STAGES, num_warps = 64, 2, 4 BLOCK_N = 64 K_TILES = 16 grid = ( @@ -421,8 +429,8 @@ def _prefill_logits( TILE_R=TILE_R, BLOCK_N=BLOCK_N, K_TILES=K_TILES, - STAGES=2, - num_warps=4, + STAGES=STAGES, + num_warps=num_warps, ) return logits @@ -521,6 +529,7 @@ def qsa_select_paged_decode( assert token_topk % compress_ratio == 0 assert block_indices.shape == (q.shape[0], token_topk // compress_ratio) assert decode_query_len > 0 and q.shape[0] % decode_query_len == 0 + assert q.dtype == k_cache.dtype, "Q and the compressed K cache must match" num_requests = q.shape[0] // decode_query_len assert page_table.shape[0] == num_requests assert visible_blocks.shape == (q.shape[0],) @@ -550,7 +559,8 @@ def qsa_select_paged_decode( BLOCK_N=_DECODE_BLOCK_N, TILES_PER_PROG=tiles_per_program, STAGES=2, - num_warps=2, + # tuned on GB300 + num_warps=1 if k_cache.dtype == torch.float8_e4m3fn else 2, ) _topk( logits, @@ -593,6 +603,7 @@ def qsa_select_paged_prefill( assert token_topk % compress_ratio == 0 assert block_indices.shape == (q.shape[0], token_topk // compress_ratio) + assert q.dtype == k_cache.dtype, "Q and the compressed K cache must match" rows = q.shape[0] # No row scores beyond cdiv(max_seq_len, compress_ratio) compressed # columns. Round up to 64 to keep the logits row stride