From c37f86e5721f089a27c8e4358c30e435805fcaab Mon Sep 17 00:00:00 2001 From: drakosha Date: Wed, 30 Sep 2026 01:36:39 +0300 Subject: [PATCH] [Bugfix] GLM-5.3-Flash: fp8 plan dtype on SM90 sparse MLA, and right-size the indexer prefill workspace (#55222) Signed-off-by: Mikhail Kostryukov Signed-off-by: NickLucche Co-authored-by: Mikhail Kostryukov Co-authored-by: NickLucche --- .../test_flashinfer_mla_sparse_sm90.py | 31 ++++++++++---- .../test_indexer_deepseek_v4_slot_mapping.py | 41 +++++++++++++++++++ vllm/models/glm5next/common/attention.py | 4 +- .../mla/flashinfer_mla_sparse_sm90.py | 8 +++- vllm/v1/attention/backends/mla/indexer.py | 19 +++++---- 5 files changed, 84 insertions(+), 19 deletions(-) diff --git a/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py b/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py index b25656cf6e..735748ae94 100644 --- a/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py +++ b/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py @@ -208,12 +208,14 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec assert builder.state.plan_calls[0][1].tolist() == expected_lens -def test_plan_uses_state_params(monkeypatch): +@pytest.mark.parametrize("kv_dtype", [torch.bfloat16, torch.float8_e4m3fn]) +def test_plan_uses_state_params(monkeypatch, kv_dtype): """The NoPE/rope dims and scale live on the builder state, not the layer. Without a device batch layout plan() takes exact per-row KV lengths and rebuilds the schedule on every call; the indptrs are always full-size - with zero-query padding rows past num_tokens. + with zero-query padding rows past num_tokens. An fp8 cache plans as + float8_e4m3fn. """ wrapper = FakeWrapper() plan_info = [0] * sm90_mod._PLAN_INFO_LEN @@ -226,7 +228,7 @@ def test_plan_uses_state_params(monkeypatch): state.device = torch.device("cpu") state.wrapper = wrapper state.num_heads = 4 - state.kv_dtype = torch.bfloat16 + state.kv_dtype = kv_dtype state.kv_lora_rank = HEAD state.qk_rope_head_dim = 64 state.sm_scale = 576**-0.5 @@ -249,7 +251,7 @@ def test_plan_uses_state_params(monkeypatch): assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False) assert scale == 576**-0.5 assert kwargs["q_data_type"] == torch.bfloat16 - assert kwargs["kv_data_type"] == torch.bfloat16 + assert kwargs["kv_data_type"] == kv_dtype # Replanning a smaller batch must clear the previous rows' lengths. state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None) @@ -339,6 +341,14 @@ def test_kv_lens_host_empty(): _NO_KPOOL = object() +@pytest.mark.parametrize( + "spec_dtype,expected_kv_dtype", + [ + (torch.bfloat16, torch.bfloat16), + (torch.uint8, torch.float8_e4m3fn), + ], + ids=["bf16", "fp8"], +) @pytest.mark.parametrize( "index_kpool,prefill_lens", [ @@ -348,9 +358,12 @@ _NO_KPOOL = object() ], ids=["kpool4", "kpool_none", "no_kpool_attr"], ) -def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens): +def test_builder_kpool_from_model_config( + monkeypatch, index_kpool, prefill_lens, spec_dtype, expected_kv_dtype +): """The builder took kpool from the KV cache spec, whose tokens_per_state is - 1, so the tail pool was never read.""" + 1, so the tail pool was never read. The state plans the cache's logical + dtype: uint8 fp8 storage becomes float8_e4m3fn.""" monkeypatch.setattr( sm90_mod.FlashInferMLASparseMetadataBuilder, "__init__", @@ -358,14 +371,15 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens) ) class _RecordingState: - def __init__(self, *_args, **kwargs): + def __init__(self, *args, **kwargs): + self.kv_dtype = args[2] self.index_topk = kwargs["index_topk"] self.index_kpool = kwargs["index_kpool"] monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState) impl, _ = make_impl(64) spec = MLAAttentionSpec( - block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16 + block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=spec_dtype ) assert spec.tokens_per_state == 1 hf_config = SimpleNamespace(index_topk=2048) @@ -384,6 +398,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens) builder = FlashInferMLASparseSM90Builder( spec, ["attn"], vllm_config, torch.device("cpu") ) + assert builder.state.kv_dtype == expected_kv_dtype # req0: prefill chunk ending at context 42295; req1: context <= topk. cam = SimpleNamespace( num_reqs=2, diff --git a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py index 9c92ae554e..bf775078c8 100644 --- a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py +++ b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py @@ -24,6 +24,7 @@ from vllm.v1.attention.backends.mla.indexer import ( DeepseekV4IndexerBackend, DeepseekV32IndexerMetadataBuilder, DeepseekV41IndexerBackend, + get_max_prefill_buffer_size, ) from vllm.v1.attention.backends.mla.sparse_utils import ( ConvertReqIndexToGlobalIndexKernel, @@ -377,3 +378,43 @@ def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_num_states(): device=device, ) torch.testing.assert_close(valid_slots, expected) + + +@pytest.mark.parametrize("compress_ratio", [1, 4]) +def test_indexer_prefill_budget_matches_compressed_workspace(compress_ratio): + """The chunker budget is in compressed rows, like the K-gather workspace.""" + max_model_len = 1024 + kv_cache_spec = MLAAttentionSpec( + block_size=256, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + tokens_per_state=compress_ratio, + ) + vllm_config = create_vllm_config(max_model_len=max_model_len) + max_num_blocks = kv_cache_spec.max_num_blocks_per_req(vllm_config, max_model_len) + block_table_width = get_block_table_width(max_num_blocks, kv_cache_spec.block_size) + builder = DeepseekV32IndexerMetadataBuilder( + kv_cache_spec=kv_cache_spec, + layer_names=["dummy"], + vllm_config=vllm_config, + device=torch.device("cpu"), + block_table_width=block_table_width, + ) + + workspace_rows = get_max_prefill_buffer_size(vllm_config) // compress_ratio + assert builder.max_prefill_buffer_size == workspace_rows + + # Overflows the workspace unless the chunker splits. + num_reqs = 3 * (workspace_rows // (max_model_len // compress_ratio)) + 1 + compressed_seq_lens = torch.full((num_reqs,), max_model_len // compress_ratio) + query_lens = torch.ones(num_reqs, dtype=torch.int64) + chunks = builder._split_indexer_prefill_chunks( + compressed_seq_lens, + query_lens, + builder.max_prefill_buffer_size, + max_logits_bytes=10**15, + ) + assert len(chunks) > 1 + for req_slice, _ in chunks: + assert int(compressed_seq_lens[req_slice].sum()) <= workspace_rows diff --git a/vllm/models/glm5next/common/attention.py b/vllm/models/glm5next/common/attention.py index 25f51117a8..f39c3a7cd0 100644 --- a/vllm/models/glm5next/common/attention.py +++ b/vllm/models/glm5next/common/attention.py @@ -312,7 +312,9 @@ class Indexer(nn.Module): self.prefix = prefix from vllm.v1.attention.backends.mla.indexer import get_max_prefill_buffer_size - self.max_total_seq_len = get_max_prefill_buffer_size(vllm_config) + self.max_total_seq_len = ( + get_max_prefill_buffer_size(vllm_config) // self.index_kpool + ) self.indexer_op = SparseAttnIndexerKpool( self.k_cache, self.quant_block_size, diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py index a5f7757d38..16d7c12e67 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py @@ -443,10 +443,16 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder): assert topk_indices_buffer is not None hf_config = vllm_config.model_config.hf_text_config assert hf_config.index_topk is not None + # fp8 KV cache is stored as uint8; plan() needs the logical dtype. + kv_plan_dtype = ( + torch.float8_e4m3fn + if kv_cache_spec.dtype == torch.uint8 + else kv_cache_spec.dtype + ) self.state = _SM90State( device, impl.num_heads, - kv_cache_spec.dtype, + kv_plan_dtype, vllm_config.scheduler_config.max_num_batched_tokens, topk_indices_buffer.shape[1], kv_lora_rank=impl.kv_lora_rank, diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index f2f2c51925..afaae5e35f 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -901,8 +901,17 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): self.use_pcp = self.pcp_world_size > 1 self.pcp_rank = get_pcp_group().rank_in_group if self.use_pcp else 0 self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size + # KV compression (DeepseekV4). Default to 1 for no compression. + self.compress_ratio = 1 + if isinstance(self.kv_cache_spec, MLAAttentionSpec): + assert isinstance(self.kv_cache_spec.tokens_per_state, int) + self.compress_ratio = self.kv_cache_spec.tokens_per_state # NOTE(Chen):an estimated max size of flattened_kv. Need to double check. - self.max_prefill_buffer_size = get_max_prefill_buffer_size(self.vllm_config) + # Counted in compressed rows, like the chunker's seq_lens and the + # workspace. + self.max_prefill_buffer_size = ( + get_max_prefill_buffer_size(self.vllm_config) // self.compress_ratio + ) self.num_speculative_tokens = ( self.vllm_config.speculative_config.num_speculative_tokens if self.vllm_config.speculative_config @@ -984,14 +993,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): (self.num_sms + 1, 2), dtype=torch.int32, device=self.device ) - # KV compression. Default to 1 for no compression. - self.compress_ratio = 1 - # Get compress_ratio for DeepseekV4 support - if isinstance(self.kv_cache_spec, MLAAttentionSpec): - # MLA compression is a whole number of tokens per state (fractions - # are whisper block pooling and never reach MLA). - assert isinstance(self.kv_cache_spec.tokens_per_state, int) - self.compress_ratio = self.kv_cache_spec.tokens_per_state if self.dcp_world_size > 1 and self.compress_ratio > 1: raise NotImplementedError( "DCP is not supported with sparse indexer KV compression "