[Perf][MRV2] Allow FULL decode graphs for one-token prompt tails (#58400)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yongye Zhu
parent
ef42093a36
commit
e7c903609f
@@ -126,7 +126,7 @@ def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch)
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
)
|
||||
|
||||
# The scheduler should mark every one of these shapes as a uniform
|
||||
@@ -262,14 +262,14 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
# The batch shape is indistinguishable from a full batch of spec decodes.
|
||||
assert (
|
||||
get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=False
|
||||
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=True
|
||||
)
|
||||
== decode_query_len
|
||||
)
|
||||
|
||||
# Two of the requests are decode_query_len tokens into a longer prompt.
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=True
|
||||
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=False
|
||||
)
|
||||
assert uniform_tok_count is None
|
||||
desc = manager.dispatch(
|
||||
@@ -283,7 +283,7 @@ def test_prompt_chunks_shaped_like_spec_decode_miss_the_full_graph(monkeypatch):
|
||||
|
||||
# The same shape with every request decoding still gets its FULL graph.
|
||||
uniform_tok_count = get_uniform_decode_token_count(
|
||||
num_reqs, num_tokens, decode_query_len, has_prefill=False
|
||||
num_reqs, num_tokens, decode_query_len, decode_graph_eligible=True
|
||||
)
|
||||
assert uniform_tok_count == decode_query_len
|
||||
desc = manager.dispatch(
|
||||
@@ -336,7 +336,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch):
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
)
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
@@ -401,7 +401,7 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch):
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
max_query_len,
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
)
|
||||
assert uniform_tok_count == max_query_len
|
||||
|
||||
|
||||
@@ -56,6 +56,7 @@ def _make_runner(
|
||||
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.decode_query_len = decode_query_len
|
||||
runner.adaptive_verification = None
|
||||
runner.pcp_manager = None
|
||||
runner.req_states = SimpleNamespace(
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(req_states)},
|
||||
# The runner keeps this as min(num_computed_tokens, prefill_len).
|
||||
@@ -128,6 +129,65 @@ def test_prompt_chunk_of_decode_query_len_is_not_uniform_decode():
|
||||
assert _uniform_token_count({"p0": (0, 8)}, 8) is None
|
||||
|
||||
|
||||
def test_one_new_prompt_token_over_context_runs_as_decode():
|
||||
# A prompt tail padded with placeholder drafts and a one-token mid-prompt
|
||||
# chunk both compute like decodes. A fresh one-token prompt and a real
|
||||
# multi-token chunk don't.
|
||||
batch = {"d0": (16, 16), "tail": (128, 129), "mid": (64, 129)}
|
||||
batch.update({"fresh": (0, 1), "chunk": (120, 129)})
|
||||
runner = _make_runner(batch, decode_query_len=4)
|
||||
scheduler_output = SimpleNamespace(
|
||||
num_scheduled_tokens={"d0": 4, "tail": 4, "mid": 1, "fresh": 1, "chunk": 4},
|
||||
total_num_scheduled_tokens=14,
|
||||
scheduled_spec_decode_tokens={"d0": [1, 2, 3], "tail": [-1] * 3},
|
||||
)
|
||||
state, _ = runner.gather_batch_req_state(scheduler_output, False)
|
||||
assert state is not None
|
||||
runs_as_decode = dict(zip(state.req_ids, state.prefill_runs_as_decode_np))
|
||||
assert runs_as_decode == {
|
||||
"d0": False,
|
||||
"tail": True,
|
||||
"mid": True,
|
||||
"fresh": False,
|
||||
"chunk": False,
|
||||
}
|
||||
|
||||
|
||||
def test_one_token_prompt_tail_with_prior_context_is_uniform_decode():
|
||||
# Replaying the last prompt token over existing context (full prefix-cache
|
||||
# hit or P/D KV import) computes exactly like a decode.
|
||||
batch = {"d0": (16, 16), "d1": (16, 16), "p0": (128, 129)}
|
||||
assert _uniform_token_count(batch, 1) == 1
|
||||
|
||||
# A fresh 1-token prompt alongside it has no prior state.
|
||||
batch["p1"] = (0, 1)
|
||||
assert _uniform_token_count(batch, 1) is None
|
||||
|
||||
# Only one-token tails qualify, not K+1 chunks with prior context.
|
||||
assert _uniform_token_count({"d0": (16, 16), "p0": (120, 129)}, 8) is None
|
||||
|
||||
|
||||
def test_padded_prompt_tail_with_prior_context_is_uniform_decode():
|
||||
# The scheduler pads a one-token prompt tail with K placeholder drafts to
|
||||
# keep the K+1 spec-decode shape; it verifies like any other decode.
|
||||
batch = {"d0": (16, 16), "d1": (16, 16), "p0": (128, 129)}
|
||||
runner = _make_runner(batch, decode_query_len=4)
|
||||
scheduler_output = SimpleNamespace(
|
||||
num_scheduled_tokens={req_id: 4 for req_id in batch},
|
||||
total_num_scheduled_tokens=12,
|
||||
scheduled_spec_decode_tokens={req_id: [-1] * 3 for req_id in batch},
|
||||
)
|
||||
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
|
||||
assert state.has_prefill and state.decode_graph_eligible
|
||||
assert uniform_tok_count == 4
|
||||
|
||||
# Without the placeholder drafts, it is a real 4-token prompt chunk.
|
||||
del scheduler_output.scheduled_spec_decode_tokens["p0"]
|
||||
state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False)
|
||||
assert not state.decode_graph_eligible
|
||||
assert uniform_tok_count is None
|
||||
|
||||
|
||||
def test_dummy_batches_stay_uniform_decode():
|
||||
# Dummy batches have no request state to consult and must stay classified
|
||||
# by shape alone; rejecting them would make capture and replay disagree.
|
||||
@@ -185,6 +245,7 @@ def test_uniform_decode_uses_state_index_not_batch_position():
|
||||
runner: Any = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.decode_query_len = 8
|
||||
runner.adaptive_verification = None
|
||||
runner.pcp_manager = None
|
||||
# State arrays in state-index order: a prefilling request, then two decodes.
|
||||
runner.req_states = SimpleNamespace(
|
||||
req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2},
|
||||
@@ -214,11 +275,11 @@ def test_uniform_decode_uses_state_index_not_batch_position():
|
||||
|
||||
|
||||
def test_uniform_decode_predicate():
|
||||
# Shape and prefill state must both pass.
|
||||
assert get_uniform_decode_token_count(2, 16, 8, False) == 8
|
||||
assert get_uniform_decode_token_count(2, 16, 8, True) is None
|
||||
# Shape and decode-graph eligibility must both pass.
|
||||
assert get_uniform_decode_token_count(2, 16, 8, True) == 8
|
||||
assert get_uniform_decode_token_count(2, 16, 8, False) is None
|
||||
# 12 tokens over 2 requests is no shared query length.
|
||||
assert get_uniform_decode_token_count(2, 12, 8, False) is None
|
||||
assert get_uniform_decode_token_count(2, 12, 8, True) is None
|
||||
|
||||
|
||||
def test_no_speculator_dispatches_on_query_length_alone():
|
||||
|
||||
@@ -96,6 +96,7 @@ def _make_batch(
|
||||
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
has_prefill=False,
|
||||
decode_graph_eligible=True,
|
||||
input_ids=torch.zeros(num_tokens, dtype=torch.int32, device=DEVICE),
|
||||
positions=torch.arange(num_tokens, dtype=torch.int64, device=DEVICE),
|
||||
is_padding=torch.zeros(num_tokens, dtype=torch.bool, device=DEVICE),
|
||||
|
||||
@@ -45,6 +45,7 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1537], dtype=torch.int32),
|
||||
is_prefilling_np=torch.tensor([False]).numpy(),
|
||||
prefill_runs_as_decode_np=None,
|
||||
dcp_local_seq_lens=None,
|
||||
positions=positions,
|
||||
prompt_lens=torch.tensor([1024], dtype=torch.int32),
|
||||
@@ -102,8 +103,7 @@ def test_padded_prompt_tail_builds_as_spec_decode(
|
||||
seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32),
|
||||
seq_lens=torch.tensor(seq_lens, dtype=torch.int32),
|
||||
is_prefilling_np=np.array(is_prefilling),
|
||||
prefill_len_np=np.array([40, 129, 100], dtype=np.int32),
|
||||
num_computed_prefill_tokens_np=np.array([40, 128, 0], dtype=np.int32),
|
||||
prefill_runs_as_decode_np=np.array([False, True, False]),
|
||||
dcp_local_seq_lens=None,
|
||||
positions=torch.zeros(12, dtype=torch.int64),
|
||||
prompt_lens=None,
|
||||
@@ -121,11 +121,12 @@ def test_padded_prompt_tail_builds_as_spec_decode(
|
||||
mamba_metadata = build_attn_metadata.call_args.kwargs[
|
||||
"model_specific_attn_metadata"
|
||||
]
|
||||
assert mamba_metadata.is_prefilling.tolist() == [False, False, True]
|
||||
|
||||
builder = _create_gdn_builder(num_speculative_tokens=k)
|
||||
common = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=seq_lens, query_lens=query_lens), BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor(is_prefilling))
|
||||
).replace(**mamba_metadata.get_extra_common_attn_kwargs(0, 3))
|
||||
meta = builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common,
|
||||
|
||||
Reference in New Issue
Block a user