[Bugfix] GLM-5.3-Flash: fp8 plan dtype on SM90 sparse MLA, and right-size the indexer prefill workspace (#55222)

Signed-off-by: Mikhail Kostryukov <mike@triptrack.net>
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: Mikhail Kostryukov <drakosha81@proton.me>
Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
drakosha
2026-09-29 22:36:39 +00:00
committed by GitHub
co-authored by Mikhail Kostryukov NickLucche
parent 50239dfc4d
commit c37f86e572
5 changed files with 84 additions and 19 deletions
@@ -208,12 +208,14 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec
assert builder.state.plan_calls[0][1].tolist() == expected_lens
def test_plan_uses_state_params(monkeypatch):
@pytest.mark.parametrize("kv_dtype", [torch.bfloat16, torch.float8_e4m3fn])
def test_plan_uses_state_params(monkeypatch, kv_dtype):
"""The NoPE/rope dims and scale live on the builder state, not the layer.
Without a device batch layout plan() takes exact per-row KV lengths and
rebuilds the schedule on every call; the indptrs are always full-size
with zero-query padding rows past num_tokens.
with zero-query padding rows past num_tokens. An fp8 cache plans as
float8_e4m3fn.
"""
wrapper = FakeWrapper()
plan_info = [0] * sm90_mod._PLAN_INFO_LEN
@@ -226,7 +228,7 @@ def test_plan_uses_state_params(monkeypatch):
state.device = torch.device("cpu")
state.wrapper = wrapper
state.num_heads = 4
state.kv_dtype = torch.bfloat16
state.kv_dtype = kv_dtype
state.kv_lora_rank = HEAD
state.qk_rope_head_dim = 64
state.sm_scale = 576**-0.5
@@ -249,7 +251,7 @@ def test_plan_uses_state_params(monkeypatch):
assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False)
assert scale == 576**-0.5
assert kwargs["q_data_type"] == torch.bfloat16
assert kwargs["kv_data_type"] == torch.bfloat16
assert kwargs["kv_data_type"] == kv_dtype
# Replanning a smaller batch must clear the previous rows' lengths.
state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None)
@@ -339,6 +341,14 @@ def test_kv_lens_host_empty():
_NO_KPOOL = object()
@pytest.mark.parametrize(
"spec_dtype,expected_kv_dtype",
[
(torch.bfloat16, torch.bfloat16),
(torch.uint8, torch.float8_e4m3fn),
],
ids=["bf16", "fp8"],
)
@pytest.mark.parametrize(
"index_kpool,prefill_lens",
[
@@ -348,9 +358,12 @@ _NO_KPOOL = object()
],
ids=["kpool4", "kpool_none", "no_kpool_attr"],
)
def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens):
def test_builder_kpool_from_model_config(
monkeypatch, index_kpool, prefill_lens, spec_dtype, expected_kv_dtype
):
"""The builder took kpool from the KV cache spec, whose tokens_per_state is
1, so the tail pool was never read."""
1, so the tail pool was never read. The state plans the cache's logical
dtype: uint8 fp8 storage becomes float8_e4m3fn."""
monkeypatch.setattr(
sm90_mod.FlashInferMLASparseMetadataBuilder,
"__init__",
@@ -358,14 +371,15 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
)
class _RecordingState:
def __init__(self, *_args, **kwargs):
def __init__(self, *args, **kwargs):
self.kv_dtype = args[2]
self.index_topk = kwargs["index_topk"]
self.index_kpool = kwargs["index_kpool"]
monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState)
impl, _ = make_impl(64)
spec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=spec_dtype
)
assert spec.tokens_per_state == 1
hf_config = SimpleNamespace(index_topk=2048)
@@ -384,6 +398,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
builder = FlashInferMLASparseSM90Builder(
spec, ["attn"], vllm_config, torch.device("cpu")
)
assert builder.state.kv_dtype == expected_kv_dtype
# req0: prefill chunk ending at context 42295; req1: context <= topk.
cam = SimpleNamespace(
num_reqs=2,
@@ -24,6 +24,7 @@ from vllm.v1.attention.backends.mla.indexer import (
DeepseekV4IndexerBackend,
DeepseekV32IndexerMetadataBuilder,
DeepseekV41IndexerBackend,
get_max_prefill_buffer_size,
)
from vllm.v1.attention.backends.mla.sparse_utils import (
ConvertReqIndexToGlobalIndexKernel,
@@ -377,3 +378,43 @@ def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_num_states():
device=device,
)
torch.testing.assert_close(valid_slots, expected)
@pytest.mark.parametrize("compress_ratio", [1, 4])
def test_indexer_prefill_budget_matches_compressed_workspace(compress_ratio):
"""The chunker budget is in compressed rows, like the K-gather workspace."""
max_model_len = 1024
kv_cache_spec = MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
tokens_per_state=compress_ratio,
)
vllm_config = create_vllm_config(max_model_len=max_model_len)
max_num_blocks = kv_cache_spec.max_num_blocks_per_req(vllm_config, max_model_len)
block_table_width = get_block_table_width(max_num_blocks, kv_cache_spec.block_size)
builder = DeepseekV32IndexerMetadataBuilder(
kv_cache_spec=kv_cache_spec,
layer_names=["dummy"],
vllm_config=vllm_config,
device=torch.device("cpu"),
block_table_width=block_table_width,
)
workspace_rows = get_max_prefill_buffer_size(vllm_config) // compress_ratio
assert builder.max_prefill_buffer_size == workspace_rows
# Overflows the workspace unless the chunker splits.
num_reqs = 3 * (workspace_rows // (max_model_len // compress_ratio)) + 1
compressed_seq_lens = torch.full((num_reqs,), max_model_len // compress_ratio)
query_lens = torch.ones(num_reqs, dtype=torch.int64)
chunks = builder._split_indexer_prefill_chunks(
compressed_seq_lens,
query_lens,
builder.max_prefill_buffer_size,
max_logits_bytes=10**15,
)
assert len(chunks) > 1
for req_slice, _ in chunks:
assert int(compressed_seq_lens[req_slice].sum()) <= workspace_rows