[MRv2]fix: model accuracy regression caused by reusing the stale last_sampled_tokens and draft_tokens (#39833)

Signed-off-by: Zijing Liu <liuzijing2014@gmail.com>
This commit is contained in:
Zijing Liu
2026-04-21 16:30:32 +00:00
committed by GitHub
parent 67eb6083e3
commit 9a6a66f3b8
+12
View File
@@ -102,6 +102,18 @@ class RequestState:
self.num_computed_prefill_tokens[req_idx] = num_computed_tokens
self.num_computed_tokens.stage_write_elem(req_idx, num_computed_tokens)
if num_computed_tokens > 0 and num_computed_tokens <= prefill_len:
# For PD disagg or resumed requests: set last_sampled to the last
# computed token so the first decode step gets the right input_id.
# For fresh prefill requests (num_computed_tokens == 0) the tensor
# is not read by combine_sampled_and_draft_tokens so we skip the
# write. Use a slice assignment rather than scalar indexing so the
# write is dispatched through fill_ without a host/device sync.
self.last_sampled_tokens[req_idx : req_idx + 1] = all_token_ids[
num_computed_tokens - 1
]
self.draft_tokens[req_idx].zero_()
def apply_staged_writes(self) -> None:
self.prompt_len.copy_to_uva()
self.prefill_len.copy_to_uva()