From 4aceabf8c1a40f8d576ba9b83e4b9a8854eda2d5 Mon Sep 17 00:00:00 2001 From: Rohan Potdar Date: Tue, 7 Jul 2026 19:22:34 -0500 Subject: [PATCH] [ROCm][Bugfix] Key sparse-MLA persistent metadata on per-request context lengths (#47766) Signed-off-by: Rohan Potdar Co-authored-by: Claude Opus 4.8 (1M context) --- .../backends/mla/rocm_aiter_mla_sparse.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index 55bba1a7ee..1e4a268306 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -522,31 +522,28 @@ class ROCMAiterMLASparseMetadataBuilder( # treated as its own batch entry), so persistent metadata can always # be precomputed here. The kernel switches to the persistent # work-stealing path automatically when work_meta_data is non-None. - # The output is a deterministic function of (num_tokens, max_query_len, - # num_heads, min(seq_lens, topk_tokens)); fingerprint those CPU-side - # and skip the launch when nothing changed. + # The output is a deterministic function of the per-request query and + # context lengths (both clamped to topk_tokens, past which per-token KV + # length saturates) and num_heads; fingerprint those CPU-side and skip + # the launch when nothing changed. num_reqs = common_attn_metadata.num_reqs clamped_seq_lens = np.minimum( common_attn_metadata.seq_lens_cpu[:num_reqs].numpy(), self.topk_tokens, ) + clamped_context_lens = np.minimum( + common_attn_metadata.seq_lens_cpu[:num_reqs].numpy() - seg_lengths, + self.topk_tokens, + ) metadata_key = ( num_tokens, int(common_attn_metadata.max_query_len), self._num_attention_heads, clamped_seq_lens.tobytes(), + clamped_context_lens.tobytes(), + seg_lengths.tobytes(), ) - # The persistent MLA kernel is numerically wrong for multi-token prefill - # batches; errors compound across chunked prefill and break long-context - # decode (vllm#47042). Use it only for decode and single-chunk prefills, - # not chunked-prefill continuations (>1 query token, seq_len > query_len). - step_query_lens = seg_lengths - total_seq_lens = common_attn_metadata.seq_lens_cpu[:num_reqs].numpy() - is_chunked_continuation = (step_query_lens > 1) & ( - total_seq_lens > step_query_lens - ) - use_persistent = not is_chunked_continuation.any() - if use_persistent and metadata_key != self._prev_metadata_key: + if metadata_key != self._prev_metadata_key: from aiter import get_mla_metadata_v1 get_mla_metadata_v1( @@ -586,7 +583,7 @@ class ROCMAiterMLASparseMetadataBuilder( paged_kv_last_page_len=paged_kv_last_page_len, paged_kv_indices=paged_kv_indices, paged_kv_indptr=paged_kv_indptr, - work_meta_data=self._mla_work_meta_data if use_persistent else None, + work_meta_data=self._mla_work_meta_data, work_indptr=self._mla_work_indptr, work_info_set=self._mla_work_info_set, reduce_indptr=self._mla_reduce_indptr,