[Bugfix] GLM-5.3-Flash: fp8 plan dtype on SM90 sparse MLA, and right-size the indexer prefill workspace (#55222)

Signed-off-by: Mikhail Kostryukov <mike@triptrack.net>
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: Mikhail Kostryukov <drakosha81@proton.me>
Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
drakosha
2026-09-29 22:36:39 +00:00
committed by GitHub
co-authored by Mikhail Kostryukov NickLucche
parent 50239dfc4d
commit c37f86e572
5 changed files with 84 additions and 19 deletions
@@ -208,12 +208,14 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec
assert builder.state.plan_calls[0][1].tolist() == expected_lens
def test_plan_uses_state_params(monkeypatch):
@pytest.mark.parametrize("kv_dtype", [torch.bfloat16, torch.float8_e4m3fn])
def test_plan_uses_state_params(monkeypatch, kv_dtype):
"""The NoPE/rope dims and scale live on the builder state, not the layer.
Without a device batch layout plan() takes exact per-row KV lengths and
rebuilds the schedule on every call; the indptrs are always full-size
with zero-query padding rows past num_tokens.
with zero-query padding rows past num_tokens. An fp8 cache plans as
float8_e4m3fn.
"""
wrapper = FakeWrapper()
plan_info = [0] * sm90_mod._PLAN_INFO_LEN
@@ -226,7 +228,7 @@ def test_plan_uses_state_params(monkeypatch):
state.device = torch.device("cpu")
state.wrapper = wrapper
state.num_heads = 4
state.kv_dtype = torch.bfloat16
state.kv_dtype = kv_dtype
state.kv_lora_rank = HEAD
state.qk_rope_head_dim = 64
state.sm_scale = 576**-0.5
@@ -249,7 +251,7 @@ def test_plan_uses_state_params(monkeypatch):
assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False)
assert scale == 576**-0.5
assert kwargs["q_data_type"] == torch.bfloat16
assert kwargs["kv_data_type"] == torch.bfloat16
assert kwargs["kv_data_type"] == kv_dtype
# Replanning a smaller batch must clear the previous rows' lengths.
state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None)
@@ -339,6 +341,14 @@ def test_kv_lens_host_empty():
_NO_KPOOL = object()
@pytest.mark.parametrize(
"spec_dtype,expected_kv_dtype",
[
(torch.bfloat16, torch.bfloat16),
(torch.uint8, torch.float8_e4m3fn),
],
ids=["bf16", "fp8"],
)
@pytest.mark.parametrize(
"index_kpool,prefill_lens",
[
@@ -348,9 +358,12 @@ _NO_KPOOL = object()
],
ids=["kpool4", "kpool_none", "no_kpool_attr"],
)
def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens):
def test_builder_kpool_from_model_config(
monkeypatch, index_kpool, prefill_lens, spec_dtype, expected_kv_dtype
):
"""The builder took kpool from the KV cache spec, whose tokens_per_state is
1, so the tail pool was never read."""
1, so the tail pool was never read. The state plans the cache's logical
dtype: uint8 fp8 storage becomes float8_e4m3fn."""
monkeypatch.setattr(
sm90_mod.FlashInferMLASparseMetadataBuilder,
"__init__",
@@ -358,14 +371,15 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
)
class _RecordingState:
def __init__(self, *_args, **kwargs):
def __init__(self, *args, **kwargs):
self.kv_dtype = args[2]
self.index_topk = kwargs["index_topk"]
self.index_kpool = kwargs["index_kpool"]
monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState)
impl, _ = make_impl(64)
spec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=spec_dtype
)
assert spec.tokens_per_state == 1
hf_config = SimpleNamespace(index_topk=2048)
@@ -384,6 +398,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
builder = FlashInferMLASparseSM90Builder(
spec, ["attn"], vllm_config, torch.device("cpu")
)
assert builder.state.kv_dtype == expected_kv_dtype
# req0: prefill chunk ending at context 42295; req1: context <= topk.
cam = SimpleNamespace(
num_reqs=2,
@@ -24,6 +24,7 @@ from vllm.v1.attention.backends.mla.indexer import (
DeepseekV4IndexerBackend,
DeepseekV32IndexerMetadataBuilder,
DeepseekV41IndexerBackend,
get_max_prefill_buffer_size,
)
from vllm.v1.attention.backends.mla.sparse_utils import (
ConvertReqIndexToGlobalIndexKernel,
@@ -377,3 +378,43 @@ def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_num_states():
device=device,
)
torch.testing.assert_close(valid_slots, expected)
@pytest.mark.parametrize("compress_ratio", [1, 4])
def test_indexer_prefill_budget_matches_compressed_workspace(compress_ratio):
"""The chunker budget is in compressed rows, like the K-gather workspace."""
max_model_len = 1024
kv_cache_spec = MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
tokens_per_state=compress_ratio,
)
vllm_config = create_vllm_config(max_model_len=max_model_len)
max_num_blocks = kv_cache_spec.max_num_blocks_per_req(vllm_config, max_model_len)
block_table_width = get_block_table_width(max_num_blocks, kv_cache_spec.block_size)
builder = DeepseekV32IndexerMetadataBuilder(
kv_cache_spec=kv_cache_spec,
layer_names=["dummy"],
vllm_config=vllm_config,
device=torch.device("cpu"),
block_table_width=block_table_width,
)
workspace_rows = get_max_prefill_buffer_size(vllm_config) // compress_ratio
assert builder.max_prefill_buffer_size == workspace_rows
# Overflows the workspace unless the chunker splits.
num_reqs = 3 * (workspace_rows // (max_model_len // compress_ratio)) + 1
compressed_seq_lens = torch.full((num_reqs,), max_model_len // compress_ratio)
query_lens = torch.ones(num_reqs, dtype=torch.int64)
chunks = builder._split_indexer_prefill_chunks(
compressed_seq_lens,
query_lens,
builder.max_prefill_buffer_size,
max_logits_bytes=10**15,
)
assert len(chunks) > 1
for req_slice, _ in chunks:
assert int(compressed_seq_lens[req_slice].sum()) <= workspace_rows
+3 -1
View File
@@ -312,7 +312,9 @@ class Indexer(nn.Module):
self.prefix = prefix
from vllm.v1.attention.backends.mla.indexer import get_max_prefill_buffer_size
self.max_total_seq_len = get_max_prefill_buffer_size(vllm_config)
self.max_total_seq_len = (
get_max_prefill_buffer_size(vllm_config) // self.index_kpool
)
self.indexer_op = SparseAttnIndexerKpool(
self.k_cache,
self.quant_block_size,
@@ -443,10 +443,16 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
assert topk_indices_buffer is not None
hf_config = vllm_config.model_config.hf_text_config
assert hf_config.index_topk is not None
# fp8 KV cache is stored as uint8; plan() needs the logical dtype.
kv_plan_dtype = (
torch.float8_e4m3fn
if kv_cache_spec.dtype == torch.uint8
else kv_cache_spec.dtype
)
self.state = _SM90State(
device,
impl.num_heads,
kv_cache_spec.dtype,
kv_plan_dtype,
vllm_config.scheduler_config.max_num_batched_tokens,
topk_indices_buffer.shape[1],
kv_lora_rank=impl.kv_lora_rank,
+10 -9
View File
@@ -901,8 +901,17 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
self.use_pcp = self.pcp_world_size > 1
self.pcp_rank = get_pcp_group().rank_in_group if self.use_pcp else 0
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
# KV compression (DeepseekV4). Default to 1 for no compression.
self.compress_ratio = 1
if isinstance(self.kv_cache_spec, MLAAttentionSpec):
assert isinstance(self.kv_cache_spec.tokens_per_state, int)
self.compress_ratio = self.kv_cache_spec.tokens_per_state
# NOTE(Chen):an estimated max size of flattened_kv. Need to double check.
self.max_prefill_buffer_size = get_max_prefill_buffer_size(self.vllm_config)
# Counted in compressed rows, like the chunker's seq_lens and the
# workspace.
self.max_prefill_buffer_size = (
get_max_prefill_buffer_size(self.vllm_config) // self.compress_ratio
)
self.num_speculative_tokens = (
self.vllm_config.speculative_config.num_speculative_tokens
if self.vllm_config.speculative_config
@@ -984,14 +993,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
(self.num_sms + 1, 2), dtype=torch.int32, device=self.device
)
# KV compression. Default to 1 for no compression.
self.compress_ratio = 1
# Get compress_ratio for DeepseekV4 support
if isinstance(self.kv_cache_spec, MLAAttentionSpec):
# MLA compression is a whole number of tokens per state (fractions
# are whisper block pooling and never reach MLA).
assert isinstance(self.kv_cache_spec.tokens_per_state, int)
self.compress_ratio = self.kv_cache_spec.tokens_per_state
if self.dcp_world_size > 1 and self.compress_ratio > 1:
raise NotImplementedError(
"DCP is not supported with sparse indexer KV compression "