[Bugfix] GLM-5.3-Flash: fp8 plan dtype on SM90 sparse MLA, and right-size the indexer prefill workspace (#55222)
Signed-off-by: Mikhail Kostryukov <mike@triptrack.net> Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Mikhail Kostryukov <drakosha81@proton.me> Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
co-authored by
Mikhail Kostryukov
NickLucche
parent
50239dfc4d
commit
c37f86e572
@@ -208,12 +208,14 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec
|
||||
assert builder.state.plan_calls[0][1].tolist() == expected_lens
|
||||
|
||||
|
||||
def test_plan_uses_state_params(monkeypatch):
|
||||
@pytest.mark.parametrize("kv_dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
def test_plan_uses_state_params(monkeypatch, kv_dtype):
|
||||
"""The NoPE/rope dims and scale live on the builder state, not the layer.
|
||||
|
||||
Without a device batch layout plan() takes exact per-row KV lengths and
|
||||
rebuilds the schedule on every call; the indptrs are always full-size
|
||||
with zero-query padding rows past num_tokens.
|
||||
with zero-query padding rows past num_tokens. An fp8 cache plans as
|
||||
float8_e4m3fn.
|
||||
"""
|
||||
wrapper = FakeWrapper()
|
||||
plan_info = [0] * sm90_mod._PLAN_INFO_LEN
|
||||
@@ -226,7 +228,7 @@ def test_plan_uses_state_params(monkeypatch):
|
||||
state.device = torch.device("cpu")
|
||||
state.wrapper = wrapper
|
||||
state.num_heads = 4
|
||||
state.kv_dtype = torch.bfloat16
|
||||
state.kv_dtype = kv_dtype
|
||||
state.kv_lora_rank = HEAD
|
||||
state.qk_rope_head_dim = 64
|
||||
state.sm_scale = 576**-0.5
|
||||
@@ -249,7 +251,7 @@ def test_plan_uses_state_params(monkeypatch):
|
||||
assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False)
|
||||
assert scale == 576**-0.5
|
||||
assert kwargs["q_data_type"] == torch.bfloat16
|
||||
assert kwargs["kv_data_type"] == torch.bfloat16
|
||||
assert kwargs["kv_data_type"] == kv_dtype
|
||||
|
||||
# Replanning a smaller batch must clear the previous rows' lengths.
|
||||
state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None)
|
||||
@@ -339,6 +341,14 @@ def test_kv_lens_host_empty():
|
||||
_NO_KPOOL = object()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"spec_dtype,expected_kv_dtype",
|
||||
[
|
||||
(torch.bfloat16, torch.bfloat16),
|
||||
(torch.uint8, torch.float8_e4m3fn),
|
||||
],
|
||||
ids=["bf16", "fp8"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"index_kpool,prefill_lens",
|
||||
[
|
||||
@@ -348,9 +358,12 @@ _NO_KPOOL = object()
|
||||
],
|
||||
ids=["kpool4", "kpool_none", "no_kpool_attr"],
|
||||
)
|
||||
def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens):
|
||||
def test_builder_kpool_from_model_config(
|
||||
monkeypatch, index_kpool, prefill_lens, spec_dtype, expected_kv_dtype
|
||||
):
|
||||
"""The builder took kpool from the KV cache spec, whose tokens_per_state is
|
||||
1, so the tail pool was never read."""
|
||||
1, so the tail pool was never read. The state plans the cache's logical
|
||||
dtype: uint8 fp8 storage becomes float8_e4m3fn."""
|
||||
monkeypatch.setattr(
|
||||
sm90_mod.FlashInferMLASparseMetadataBuilder,
|
||||
"__init__",
|
||||
@@ -358,14 +371,15 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
|
||||
)
|
||||
|
||||
class _RecordingState:
|
||||
def __init__(self, *_args, **kwargs):
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.kv_dtype = args[2]
|
||||
self.index_topk = kwargs["index_topk"]
|
||||
self.index_kpool = kwargs["index_kpool"]
|
||||
|
||||
monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState)
|
||||
impl, _ = make_impl(64)
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=spec_dtype
|
||||
)
|
||||
assert spec.tokens_per_state == 1
|
||||
hf_config = SimpleNamespace(index_topk=2048)
|
||||
@@ -384,6 +398,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
|
||||
builder = FlashInferMLASparseSM90Builder(
|
||||
spec, ["attn"], vllm_config, torch.device("cpu")
|
||||
)
|
||||
assert builder.state.kv_dtype == expected_kv_dtype
|
||||
# req0: prefill chunk ending at context 42295; req1: context <= topk.
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV4IndexerBackend,
|
||||
DeepseekV32IndexerMetadataBuilder,
|
||||
DeepseekV41IndexerBackend,
|
||||
get_max_prefill_buffer_size,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
ConvertReqIndexToGlobalIndexKernel,
|
||||
@@ -377,3 +378,43 @@ def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_num_states():
|
||||
device=device,
|
||||
)
|
||||
torch.testing.assert_close(valid_slots, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 4])
|
||||
def test_indexer_prefill_budget_matches_compressed_workspace(compress_ratio):
|
||||
"""The chunker budget is in compressed rows, like the K-gather workspace."""
|
||||
max_model_len = 1024
|
||||
kv_cache_spec = MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=compress_ratio,
|
||||
)
|
||||
vllm_config = create_vllm_config(max_model_len=max_model_len)
|
||||
max_num_blocks = kv_cache_spec.max_num_blocks_per_req(vllm_config, max_model_len)
|
||||
block_table_width = get_block_table_width(max_num_blocks, kv_cache_spec.block_size)
|
||||
builder = DeepseekV32IndexerMetadataBuilder(
|
||||
kv_cache_spec=kv_cache_spec,
|
||||
layer_names=["dummy"],
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cpu"),
|
||||
block_table_width=block_table_width,
|
||||
)
|
||||
|
||||
workspace_rows = get_max_prefill_buffer_size(vllm_config) // compress_ratio
|
||||
assert builder.max_prefill_buffer_size == workspace_rows
|
||||
|
||||
# Overflows the workspace unless the chunker splits.
|
||||
num_reqs = 3 * (workspace_rows // (max_model_len // compress_ratio)) + 1
|
||||
compressed_seq_lens = torch.full((num_reqs,), max_model_len // compress_ratio)
|
||||
query_lens = torch.ones(num_reqs, dtype=torch.int64)
|
||||
chunks = builder._split_indexer_prefill_chunks(
|
||||
compressed_seq_lens,
|
||||
query_lens,
|
||||
builder.max_prefill_buffer_size,
|
||||
max_logits_bytes=10**15,
|
||||
)
|
||||
assert len(chunks) > 1
|
||||
for req_slice, _ in chunks:
|
||||
assert int(compressed_seq_lens[req_slice].sum()) <= workspace_rows
|
||||
|
||||
@@ -312,7 +312,9 @@ class Indexer(nn.Module):
|
||||
self.prefix = prefix
|
||||
from vllm.v1.attention.backends.mla.indexer import get_max_prefill_buffer_size
|
||||
|
||||
self.max_total_seq_len = get_max_prefill_buffer_size(vllm_config)
|
||||
self.max_total_seq_len = (
|
||||
get_max_prefill_buffer_size(vllm_config) // self.index_kpool
|
||||
)
|
||||
self.indexer_op = SparseAttnIndexerKpool(
|
||||
self.k_cache,
|
||||
self.quant_block_size,
|
||||
|
||||
@@ -443,10 +443,16 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
|
||||
assert topk_indices_buffer is not None
|
||||
hf_config = vllm_config.model_config.hf_text_config
|
||||
assert hf_config.index_topk is not None
|
||||
# fp8 KV cache is stored as uint8; plan() needs the logical dtype.
|
||||
kv_plan_dtype = (
|
||||
torch.float8_e4m3fn
|
||||
if kv_cache_spec.dtype == torch.uint8
|
||||
else kv_cache_spec.dtype
|
||||
)
|
||||
self.state = _SM90State(
|
||||
device,
|
||||
impl.num_heads,
|
||||
kv_cache_spec.dtype,
|
||||
kv_plan_dtype,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
topk_indices_buffer.shape[1],
|
||||
kv_lora_rank=impl.kv_lora_rank,
|
||||
|
||||
@@ -901,8 +901,17 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
self.use_pcp = self.pcp_world_size > 1
|
||||
self.pcp_rank = get_pcp_group().rank_in_group if self.use_pcp else 0
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
# KV compression (DeepseekV4). Default to 1 for no compression.
|
||||
self.compress_ratio = 1
|
||||
if isinstance(self.kv_cache_spec, MLAAttentionSpec):
|
||||
assert isinstance(self.kv_cache_spec.tokens_per_state, int)
|
||||
self.compress_ratio = self.kv_cache_spec.tokens_per_state
|
||||
# NOTE(Chen):an estimated max size of flattened_kv. Need to double check.
|
||||
self.max_prefill_buffer_size = get_max_prefill_buffer_size(self.vllm_config)
|
||||
# Counted in compressed rows, like the chunker's seq_lens and the
|
||||
# workspace.
|
||||
self.max_prefill_buffer_size = (
|
||||
get_max_prefill_buffer_size(self.vllm_config) // self.compress_ratio
|
||||
)
|
||||
self.num_speculative_tokens = (
|
||||
self.vllm_config.speculative_config.num_speculative_tokens
|
||||
if self.vllm_config.speculative_config
|
||||
@@ -984,14 +993,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
(self.num_sms + 1, 2), dtype=torch.int32, device=self.device
|
||||
)
|
||||
|
||||
# KV compression. Default to 1 for no compression.
|
||||
self.compress_ratio = 1
|
||||
# Get compress_ratio for DeepseekV4 support
|
||||
if isinstance(self.kv_cache_spec, MLAAttentionSpec):
|
||||
# MLA compression is a whole number of tokens per state (fractions
|
||||
# are whisper block pooling and never reach MLA).
|
||||
assert isinstance(self.kv_cache_spec.tokens_per_state, int)
|
||||
self.compress_ratio = self.kv_cache_spec.tokens_per_state
|
||||
if self.dcp_world_size > 1 and self.compress_ratio > 1:
|
||||
raise NotImplementedError(
|
||||
"DCP is not supported with sparse indexer KV compression "
|
||||
|
||||
Reference in New Issue
Block a user