[Bugfix] Fix token loss in PP mode which causes degraded accuracy (#41133)

Signed-off-by: Jing Wang <jingwang96@qq.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jing Wang
2026-05-06 14:07:32 -04:00
committed by GitHub
co-authored by Copilot Wentao Ye mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 22a3cbe152
commit 27702f6d08
3 changed files with 179 additions and 7 deletions
+1
View File
@@ -504,6 +504,7 @@ class InputBatch:
start_index = self.num_tokens_no_spec[req_index]
end_token_index = start_index + num_spec_tokens
self.token_ids_cpu[req_index, start_index:end_token_index] = spec_token_ids
self.is_token_ids[req_index, start_index:end_token_index] = True
cur_spec_token_ids.extend(spec_token_ids)
def remove_request(self, req_id: str) -> int | None:
+21 -7
View File
@@ -1347,13 +1347,27 @@ class GPUModelRunner(
# For the last rank, we don't need to update the token_ids_cpu
# because the sampled tokens are already cached.
if not is_last_rank:
# Add new_token_ids to token_ids_cpu.
start_token_index = num_computed_tokens
end_token_index = num_computed_tokens + len(new_token_ids)
self.input_batch.token_ids_cpu[
req_index, start_token_index:end_token_index
] = new_token_ids
self.input_batch.num_tokens_no_spec[req_index] = end_token_index
start_token_index = self.input_batch.num_tokens_no_spec[req_index]
# For chunked prefill, num_computed_tokens may less
# than num_tokens_no_spec.
# Async scheduled PP: no new_token_ids, advance num_tokens_no_spec
# according to num_computed_tokens.
end_token_index = max(
start_token_index,
num_computed_tokens + len(new_token_ids),
)
if end_token_index > start_token_index:
if new_token_ids:
# Add new_token_ids to token_ids_cpu.
num_new_tokens = end_token_index - start_token_index
tokens_to_append = new_token_ids[-num_new_tokens:]
self.input_batch.token_ids_cpu[
req_index, start_token_index:end_token_index
] = tokens_to_append
self.input_batch.is_token_ids[
req_index, start_token_index:end_token_index
] = True
self.input_batch.num_tokens_no_spec[req_index] = end_token_index
# Add spec_token_ids to token_ids_cpu.
self.input_batch.update_req_spec_token_ids(req_state, scheduled_spec_tokens)