[Perf][MRV2] Skip empty draft-token resets on request admission (#61640)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Nick Hill
parent
c1407a7e86
commit
857ddac4e5
@@ -114,7 +114,8 @@ class RequestState:
|
||||
self.num_computed_tokens_np[req_idx] = num_computed_tokens
|
||||
self.num_computed_tokens.stage_write_elem(req_idx, num_computed_tokens)
|
||||
|
||||
self.draft_tokens[req_idx].zero_()
|
||||
if self.num_speculative_steps > 0:
|
||||
self.draft_tokens[req_idx].zero_()
|
||||
|
||||
def apply_staged_writes(self) -> None:
|
||||
self.prompt_len.copy_to_uva()
|
||||
|
||||
Reference in New Issue
Block a user