[Bugfix] Fix token loss in PP mode which causes degraded accuracy (#41133)
Signed-off-by: Jing Wang <jingwang96@qq.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Copilot
Wentao Ye
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
22a3cbe152
commit
27702f6d08
@@ -504,6 +504,7 @@ class InputBatch:
|
||||
start_index = self.num_tokens_no_spec[req_index]
|
||||
end_token_index = start_index + num_spec_tokens
|
||||
self.token_ids_cpu[req_index, start_index:end_token_index] = spec_token_ids
|
||||
self.is_token_ids[req_index, start_index:end_token_index] = True
|
||||
cur_spec_token_ids.extend(spec_token_ids)
|
||||
|
||||
def remove_request(self, req_id: str) -> int | None:
|
||||
|
||||
Reference in New Issue
Block a user