[bugfix] Fix prompt logprobs on request eviction during chunked prefill (#41411)
Signed-off-by: Joachim Studnia <joachim@mistral.ai> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
8c780943b4
commit
422dd02598
@@ -49,6 +49,8 @@ class CachedRequestState:
|
||||
|
||||
lora_request: LoRARequest | None = None
|
||||
prompt_embeds: torch.Tensor | None = None
|
||||
# To accumulate prompt logprobs tensor chunks across prefill steps.
|
||||
in_progress_prompt_logprobs_cpu: LogprobsTensors | None = None
|
||||
|
||||
# Per-position mask for mixed-mode inputs (e.g chat completion with
|
||||
# prompt_embeds content parts). See `Request.prompt_is_token_ids`.
|
||||
@@ -255,9 +257,6 @@ class InputBatch:
|
||||
# More efficient than num_logprobs=-1 when only a few tokens are needed
|
||||
self.logprob_token_ids: dict[str, list[int]] = {}
|
||||
|
||||
# To accumulate prompt logprobs tensor chunks across prefill steps.
|
||||
self.in_progress_prompt_logprobs_cpu: dict[str, LogprobsTensors] = {}
|
||||
|
||||
# Internal representation of per-step batch state changes, used for
|
||||
# reordering persistent batch and generating logitsprocs batch state
|
||||
# updates. Should reset each step.
|
||||
@@ -552,7 +551,6 @@ class InputBatch:
|
||||
self.generators.pop(req_index, None)
|
||||
self.num_logprobs.pop(req_id, None)
|
||||
self.logprob_token_ids.pop(req_id, None)
|
||||
self.in_progress_prompt_logprobs_cpu.pop(req_id, None)
|
||||
if self.prev_req_id_to_index is not None:
|
||||
self.prev_req_id_to_index.pop(req_id, None)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user