[Perf][MRV2] Allow FULL decode graphs for one-token prompt tails (#58400)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yongye Zhu
parent
ef42093a36
commit
e7c903609f
@@ -126,7 +126,7 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
)
|
||||
|
||||
# The scheduler should mark every one of these shapes as a uniform
|
||||
@@ -262,14 +262,14 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
# The batch shape is indistinguishable from a full batch of spec decodes.
|
||||
assert (
|
||||
get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=False
|
||||
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=True
|
||||
)
|
||||
== decode_query_len
|
||||
)
|
||||
|
||||
# Two of the requests are decode_query_len tokens into a longer prompt.
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=True
|
||||
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=False
|
||||
)
|
||||
assert uniform_tok_count is None
|
||||
desc = manager.dispatch(
|
||||
@@ -283,7 +283,7 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
|
||||
# The same shape with every request decoding still gets its FULL graph.
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=False
|
||||
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=True
|
||||
)
|
||||
assert uniform_tok_count == decode_query_len
|
||||
desc = manager.dispatch(
|
||||
@@ -336,7 +336,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
)
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
@@ -401,7 +401,7 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
)
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
|
||||
Reference in New Issue
Block a user