[bugfix] Fix prompt logprobs on request eviction during chunked prefill (#41411)

Signed-off-by: Joachim Studnia <joachim@mistral.ai>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joachim Studnia
2026-05-04 11:46:00 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 8c780943b4
commit 422dd02598
5 changed files with 19 additions and 13 deletions
+2 -4
View File
@@ -49,6 +49,8 @@ class CachedRequestState:
lora_request: LoRARequest | None = None
prompt_embeds: torch.Tensor | None = None
# To accumulate prompt logprobs tensor chunks across prefill steps.
in_progress_prompt_logprobs_cpu: LogprobsTensors | None = None
# Per-position mask for mixed-mode inputs (e.g chat completion with
# prompt_embeds content parts). See `Request.prompt_is_token_ids`.
@@ -255,9 +257,6 @@ class InputBatch:
# More efficient than num_logprobs=-1 when only a few tokens are needed
self.logprob_token_ids: dict[str, list[int]] = {}
# To accumulate prompt logprobs tensor chunks across prefill steps.
self.in_progress_prompt_logprobs_cpu: dict[str, LogprobsTensors] = {}
# Internal representation of per-step batch state changes, used for
# reordering persistent batch and generating logitsprocs batch state
# updates. Should reset each step.
@@ -552,7 +551,6 @@ class InputBatch:
self.generators.pop(req_index, None)
self.num_logprobs.pop(req_id, None)
self.logprob_token_ids.pop(req_id, None)
self.in_progress_prompt_logprobs_cpu.pop(req_id, None)
if self.prev_req_id_to_index is not None:
self.prev_req_id_to_index.pop(req_id, None)