[bugfix] Fix prompt logprobs on request eviction during chunked prefill (#41411)
Signed-off-by: Joachim Studnia <joachim@mistral.ai> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
8c780943b4
commit
422dd02598
@@ -996,6 +996,8 @@ class VllmRunner:
|
||||
req_sample_output_ids: list[list[int]] = []
|
||||
req_sample_output_strs: list[str] = []
|
||||
req_logprobs = []
|
||||
if req_output.prompt_logprobs:
|
||||
req_logprobs.extend(req_output.prompt_logprobs)
|
||||
for sample in req_output.outputs:
|
||||
output_str = sample.text
|
||||
output_ids = list(sample.token_ids)
|
||||
|
||||
@@ -57,6 +57,8 @@ def test_without_spec_decoding(
|
||||
dict(bad_words=["the", " the"]),
|
||||
dict(logprobs=2),
|
||||
dict(logprobs=2, frequency_penalty=-1.0),
|
||||
dict(prompt_logprobs=2),
|
||||
dict(prompt_logprobs=2, logprobs=2),
|
||||
dict(structured_outputs=struct_outputs),
|
||||
dict(
|
||||
structured_outputs=struct_outputs,
|
||||
@@ -126,6 +128,8 @@ def test_with_eagle3_spec_decoding(sample_json_schema, monkeypatch: pytest.Monke
|
||||
dict(bad_words=["the", " the"]),
|
||||
dict(logprobs=2),
|
||||
dict(logprobs=2, frequency_penalty=-1.0),
|
||||
dict(prompt_logprobs=2),
|
||||
dict(prompt_logprobs=2, logprobs=2),
|
||||
dict(structured_outputs=struct_outputs),
|
||||
dict(
|
||||
structured_outputs=struct_outputs,
|
||||
@@ -413,7 +417,12 @@ def _all_logprobs_match(req_a, req_b) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _logprobs_match(lps_a: dict[int, Logprob], lps_b: dict[int, Logprob]) -> bool:
|
||||
def _logprobs_match(
|
||||
lps_a: dict[int, Logprob] | None,
|
||||
lps_b: dict[int, Logprob] | None,
|
||||
) -> bool:
|
||||
if lps_a is None or lps_b is None:
|
||||
return lps_a is lps_b
|
||||
rel_tol, abs_tol = 1e-3, 1e-6
|
||||
return (
|
||||
len(lps_a) == len(lps_b)
|
||||
|
||||
Reference in New Issue
Block a user