[Perf][MRV2] Allow FULL decode graphs for one-token prompt tails (#58400)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yongye Zhu
parent
ef42093a36
commit
e7c903609f
@@ -1242,7 +1242,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if dummy_run:
|
||||
# Dummy batches are uniform by construction.
|
||||
return None, get_uniform_decode_token_count(
|
||||
num_reqs, num_toks, max_query_len, has_prefill=False
|
||||
num_reqs, num_toks, max_query_len, decode_graph_eligible=True
|
||||
)
|
||||
|
||||
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
@@ -1261,24 +1261,48 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
idx_mapping_np
|
||||
]
|
||||
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
|
||||
has_prefill = bool(is_prefilling_np.any())
|
||||
|
||||
if self.adaptive_verification is not None and draft_tokens:
|
||||
num_toks = self.adaptive_verification.get_num_tokens(
|
||||
num_tokens_per_req, draft_tokens
|
||||
num_draft_tokens_np = None
|
||||
if draft_tokens:
|
||||
num_drafts_iter = (len(draft_tokens.get(req_id, ())) for req_id in req_ids)
|
||||
num_draft_tokens_np = np.fromiter(
|
||||
num_drafts_iter, dtype=np.int32, count=num_reqs
|
||||
)
|
||||
if self.adaptive_verification is not None:
|
||||
num_toks = self.adaptive_verification.get_num_tokens(
|
||||
num_tokens_per_req, draft_tokens
|
||||
)
|
||||
|
||||
prefill_runs_as_decode_np = None
|
||||
decode_graph_eligible = not has_prefill
|
||||
if has_prefill and self.pcp_manager is None:
|
||||
# One new prompt token over existing context (possibly padded with
|
||||
# placeholder drafts) runs as a decode. PCP partitions prefill rows.
|
||||
num_new_tokens = num_scheduled_tokens
|
||||
if num_draft_tokens_np is not None:
|
||||
num_new_tokens = num_new_tokens - num_draft_tokens_np
|
||||
one_token_ext = (num_new_tokens == 1) & (num_computed_prefill_tokens_np > 0)
|
||||
prefill_runs_as_decode_np = is_prefilling_np & one_token_ext
|
||||
decode_graph_eligible = bool(
|
||||
(prefill_runs_as_decode_np | ~is_prefilling_np).all()
|
||||
)
|
||||
|
||||
batch_state = BatchReqState(
|
||||
req_ids=req_ids,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
num_tokens=num_toks,
|
||||
num_draft_tokens_np=num_draft_tokens_np,
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
prefill_len_np=prefill_len_np,
|
||||
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=is_prefilling_np,
|
||||
has_prefill=bool(is_prefilling_np.any()),
|
||||
has_prefill=has_prefill,
|
||||
prefill_runs_as_decode_np=prefill_runs_as_decode_np,
|
||||
decode_graph_eligible=decode_graph_eligible,
|
||||
)
|
||||
return batch_state, get_uniform_decode_token_count(
|
||||
num_reqs, num_toks, max_query_len, batch_state.has_prefill
|
||||
num_reqs, num_toks, max_query_len, decode_graph_eligible
|
||||
)
|
||||
|
||||
def prepare_inputs(
|
||||
@@ -1320,11 +1344,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_reqs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
else:
|
||||
num_draft_tokens_per_req = np.fromiter(
|
||||
(len(draft_tokens.get(req_id, ())) for req_id in req_ids),
|
||||
dtype=np.int32,
|
||||
count=num_reqs,
|
||||
)
|
||||
num_draft_tokens_per_req = batch_req_state.num_draft_tokens_np
|
||||
if num_draft_tokens_per_req is None:
|
||||
num_draft_tokens_per_req = np.zeros(num_reqs, dtype=np.int32)
|
||||
num_bonus_tokens = self.model_state.num_new_sampled_tokens_per_step
|
||||
total_num_draft_tokens = int(num_draft_tokens_per_req.sum())
|
||||
total_num_logits = num_reqs * num_bonus_tokens + total_num_draft_tokens
|
||||
@@ -1463,6 +1485,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=batch_req_state.is_prefilling_np,
|
||||
has_prefill=batch_req_state.has_prefill,
|
||||
prefill_runs_as_decode_np=batch_req_state.prefill_runs_as_decode_np,
|
||||
decode_graph_eligible=batch_req_state.decode_graph_eligible,
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
||||
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
||||
is_padding=is_padding,
|
||||
@@ -2345,11 +2369,14 @@ class BatchReqState(NamedTuple):
|
||||
# May be less than scheduler_output.total_num_scheduled_tokens:
|
||||
# adaptive verification trims the draft budget before running.
|
||||
num_tokens: int
|
||||
num_draft_tokens_np: np.ndarray | None # [num_reqs], None if no drafts
|
||||
idx_mapping_np: np.ndarray # [num_reqs]
|
||||
prefill_len_np: np.ndarray # [num_reqs]
|
||||
num_computed_prefill_tokens_np: np.ndarray # [num_reqs]
|
||||
is_prefilling_np: np.ndarray # [num_reqs]
|
||||
has_prefill: bool
|
||||
prefill_runs_as_decode_np: np.ndarray | None # [num_reqs]
|
||||
decode_graph_eligible: bool
|
||||
|
||||
|
||||
def sort_batch_req_ids(
|
||||
|
||||
Reference in New Issue
Block a user