[Bugfix] Fix token loss in PP mode which causes degraded accuracy (#41133)
Signed-off-by: Jing Wang <jingwang96@qq.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Copilot
Wentao Ye
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
22a3cbe152
commit
27702f6d08
@@ -504,6 +504,7 @@ class InputBatch:
|
||||
start_index = self.num_tokens_no_spec[req_index]
|
||||
end_token_index = start_index + num_spec_tokens
|
||||
self.token_ids_cpu[req_index, start_index:end_token_index] = spec_token_ids
|
||||
self.is_token_ids[req_index, start_index:end_token_index] = True
|
||||
cur_spec_token_ids.extend(spec_token_ids)
|
||||
|
||||
def remove_request(self, req_id: str) -> int | None:
|
||||
|
||||
@@ -1347,13 +1347,27 @@ class GPUModelRunner(
|
||||
# For the last rank, we don't need to update the token_ids_cpu
|
||||
# because the sampled tokens are already cached.
|
||||
if not is_last_rank:
|
||||
# Add new_token_ids to token_ids_cpu.
|
||||
start_token_index = num_computed_tokens
|
||||
end_token_index = num_computed_tokens + len(new_token_ids)
|
||||
self.input_batch.token_ids_cpu[
|
||||
req_index, start_token_index:end_token_index
|
||||
] = new_token_ids
|
||||
self.input_batch.num_tokens_no_spec[req_index] = end_token_index
|
||||
start_token_index = self.input_batch.num_tokens_no_spec[req_index]
|
||||
# For chunked prefill, num_computed_tokens may less
|
||||
# than num_tokens_no_spec.
|
||||
# Async scheduled PP: no new_token_ids, advance num_tokens_no_spec
|
||||
# according to num_computed_tokens.
|
||||
end_token_index = max(
|
||||
start_token_index,
|
||||
num_computed_tokens + len(new_token_ids),
|
||||
)
|
||||
if end_token_index > start_token_index:
|
||||
if new_token_ids:
|
||||
# Add new_token_ids to token_ids_cpu.
|
||||
num_new_tokens = end_token_index - start_token_index
|
||||
tokens_to_append = new_token_ids[-num_new_tokens:]
|
||||
self.input_batch.token_ids_cpu[
|
||||
req_index, start_token_index:end_token_index
|
||||
] = tokens_to_append
|
||||
self.input_batch.is_token_ids[
|
||||
req_index, start_token_index:end_token_index
|
||||
] = True
|
||||
self.input_batch.num_tokens_no_spec[req_index] = end_token_index
|
||||
|
||||
# Add spec_token_ids to token_ids_cpu.
|
||||
self.input_batch.update_req_spec_token_ids(req_state, scheduled_spec_tokens)
|
||||
|
||||
Reference in New Issue
Block a user