[ROCm][Bugfix] Key sparse-MLA persistent metadata on per-request context lengths (#47766)
Signed-off-by: Rohan Potdar <rohan.potdar@amd.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
6e35c5e5af
commit
4aceabf8c1
@@ -522,31 +522,28 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
# treated as its own batch entry), so persistent metadata can always
|
||||
# be precomputed here. The kernel switches to the persistent
|
||||
# work-stealing path automatically when work_meta_data is non-None.
|
||||
# The output is a deterministic function of (num_tokens, max_query_len,
|
||||
# num_heads, min(seq_lens, topk_tokens)); fingerprint those CPU-side
|
||||
# and skip the launch when nothing changed.
|
||||
# The output is a deterministic function of the per-request query and
|
||||
# context lengths (both clamped to topk_tokens, past which per-token KV
|
||||
# length saturates) and num_heads; fingerprint those CPU-side and skip
|
||||
# the launch when nothing changed.
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
clamped_seq_lens = np.minimum(
|
||||
common_attn_metadata.seq_lens_cpu[:num_reqs].numpy(),
|
||||
self.topk_tokens,
|
||||
)
|
||||
clamped_context_lens = np.minimum(
|
||||
common_attn_metadata.seq_lens_cpu[:num_reqs].numpy() - seg_lengths,
|
||||
self.topk_tokens,
|
||||
)
|
||||
metadata_key = (
|
||||
num_tokens,
|
||||
int(common_attn_metadata.max_query_len),
|
||||
self._num_attention_heads,
|
||||
clamped_seq_lens.tobytes(),
|
||||
clamped_context_lens.tobytes(),
|
||||
seg_lengths.tobytes(),
|
||||
)
|
||||
# The persistent MLA kernel is numerically wrong for multi-token prefill
|
||||
# batches; errors compound across chunked prefill and break long-context
|
||||
# decode (vllm#47042). Use it only for decode and single-chunk prefills,
|
||||
# not chunked-prefill continuations (>1 query token, seq_len > query_len).
|
||||
step_query_lens = seg_lengths
|
||||
total_seq_lens = common_attn_metadata.seq_lens_cpu[:num_reqs].numpy()
|
||||
is_chunked_continuation = (step_query_lens > 1) & (
|
||||
total_seq_lens > step_query_lens
|
||||
)
|
||||
use_persistent = not is_chunked_continuation.any()
|
||||
if use_persistent and metadata_key != self._prev_metadata_key:
|
||||
if metadata_key != self._prev_metadata_key:
|
||||
from aiter import get_mla_metadata_v1
|
||||
|
||||
get_mla_metadata_v1(
|
||||
@@ -586,7 +583,7 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
paged_kv_last_page_len=paged_kv_last_page_len,
|
||||
paged_kv_indices=paged_kv_indices,
|
||||
paged_kv_indptr=paged_kv_indptr,
|
||||
work_meta_data=self._mla_work_meta_data if use_persistent else None,
|
||||
work_meta_data=self._mla_work_meta_data,
|
||||
work_indptr=self._mla_work_indptr,
|
||||
work_info_set=self._mla_work_info_set,
|
||||
reduce_indptr=self._mla_reduce_indptr,
|
||||
|
||||
Reference in New Issue
Block a user