[Perf][MRV2] Allow FULL decode graphs for one-token prompt tails (#58400)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yongye Zhu
parent
ef42093a36
commit
e7c903609f
@@ -747,10 +747,12 @@ def is_uniform_query_len(num_reqs: int, num_tokens: int, max_query_len: int) ->
|
||||
|
||||
|
||||
def get_uniform_decode_token_count(
|
||||
num_reqs: int, num_tokens: int, max_query_len: int, has_prefill: bool
|
||||
num_reqs: int, num_tokens: int, max_query_len: int, decode_graph_eligible: bool
|
||||
) -> int | None:
|
||||
"""Per-request token count of a uniform decode batch, or None."""
|
||||
if not has_prefill and is_uniform_query_len(num_reqs, num_tokens, max_query_len):
|
||||
if decode_graph_eligible and is_uniform_query_len(
|
||||
num_reqs, num_tokens, max_query_len
|
||||
):
|
||||
return max_query_len
|
||||
return None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user