[ROCm][MLA] Cut ~21x host dispatch from the MLA metadata build (#58381)
Signed-off-by: lifulu <fululi12@amd.com> Signed-off-by: lifulu <lifulu@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: lifulu <lifulu@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
lifulu
parent
c9e62f13c2
commit
8055293353
@@ -2540,16 +2540,26 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
"Non-causal multi-token MLA requires an explicitly supported "
|
||||
"attention group."
|
||||
)
|
||||
query_lens = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
num_active_reqs = int(torch.count_nonzero(query_lens > 0))
|
||||
uniform_active_queries = num_active_reqs > 0 and bool(
|
||||
torch.all(query_lens[:num_active_reqs] == query_lens[0])
|
||||
# query_start_loc_cpu is host-side and holds one entry per
|
||||
# request, so this check costs more in tensor dispatch than in
|
||||
# the work it does. Read it out once and validate on Python ints:
|
||||
# the block runs on every build, between decode steps.
|
||||
qsl = query_start_loc_cpu.tolist()
|
||||
query_lens = [qsl[i + 1] - qsl[i] for i in range(len(qsl) - 1)]
|
||||
num_active_reqs = 0
|
||||
for qlen in query_lens:
|
||||
if qlen > 0:
|
||||
num_active_reqs += 1
|
||||
uniform_active_queries = num_active_reqs > 0 and all(
|
||||
qlen == query_lens[0] for qlen in query_lens[:num_active_reqs]
|
||||
)
|
||||
trailing_graph_padding = all(
|
||||
qlen == 0 for qlen in query_lens[num_active_reqs:]
|
||||
)
|
||||
trailing_graph_padding = bool(torch.all(query_lens[num_active_reqs:] == 0))
|
||||
if not (uniform_active_queries and trailing_graph_padding):
|
||||
raise ValueError(
|
||||
"Non-causal MLA requires a uniform query block; got query "
|
||||
f"lengths {query_lens.tolist()}."
|
||||
f"lengths {query_lens}."
|
||||
)
|
||||
# Use exact GPU sequence lengths instead of the prefill path's CPU
|
||||
# context-length upper bounds.
|
||||
|
||||
Reference in New Issue
Block a user