[bugfix] Fix prompt logprobs on request eviction during chunked prefill (#41411)

Signed-off-by: Joachim Studnia <joachim@mistral.ai>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joachim Studnia
2026-05-04 11:46:00 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 8c780943b4
commit 422dd02598
5 changed files with 19 additions and 13 deletions
+2
View File
@@ -996,6 +996,8 @@ class VllmRunner:
req_sample_output_ids: list[list[int]] = []
req_sample_output_strs: list[str] = []
req_logprobs = []
if req_output.prompt_logprobs:
req_logprobs.extend(req_output.prompt_logprobs)
for sample in req_output.outputs:
output_str = sample.text
output_ids = list(sample.token_ids)
+10 -1
View File
@@ -57,6 +57,8 @@ def test_without_spec_decoding(
dict(bad_words=["the", " the"]),
dict(logprobs=2),
dict(logprobs=2, frequency_penalty=-1.0),
dict(prompt_logprobs=2),
dict(prompt_logprobs=2, logprobs=2),
dict(structured_outputs=struct_outputs),
dict(
structured_outputs=struct_outputs,
@@ -126,6 +128,8 @@ def test_with_eagle3_spec_decoding(sample_json_schema, monkeypatch: pytest.Monke
dict(bad_words=["the", " the"]),
dict(logprobs=2),
dict(logprobs=2, frequency_penalty=-1.0),
dict(prompt_logprobs=2),
dict(prompt_logprobs=2, logprobs=2),
dict(structured_outputs=struct_outputs),
dict(
structured_outputs=struct_outputs,
@@ -413,7 +417,12 @@ def _all_logprobs_match(req_a, req_b) -> bool:
)
def _logprobs_match(lps_a: dict[int, Logprob], lps_b: dict[int, Logprob]) -> bool:
def _logprobs_match(
lps_a: dict[int, Logprob] | None,
lps_b: dict[int, Logprob] | None,
) -> bool:
if lps_a is None or lps_b is None:
return lps_a is lps_b
rel_tol, abs_tol = 1e-3, 1e-6
return (
len(lps_a) == len(lps_b)