[Feature] Add fixed-token prefill scoring (#54335)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-28 07:50:09 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5
parent a7ff44355c
commit b721a4c709
36 changed files with 768 additions and 8 deletions
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import json
import httpx
import numpy as np
import pybase64 as base64
import pytest
import pytest_asyncio
from transformers import AutoTokenizer
@@ -311,6 +314,52 @@ async def test_generate_logprobs(client, logprobs_value):
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
@pytest.mark.asyncio
async def test_generate_prompt_token_id_logprobs(client):
token_ids = [11, 22, 33, 44, 55]
candidates = [22, 33, 44, 55]
sampling_params = {
"max_tokens": 1,
"prompt_logprobs": 0,
"prompt_logprob_token_ids": candidates,
"prompt_logprob_start": 1,
}
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": sampling_params,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
assert scores.shape == (len(token_ids) - 2, len(candidates))
for row, target in enumerate(token_ids[2:]):
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
assert scores[row, candidates.index(target)] == pytest.approx(
expected, abs=1e-3
)
stream_only_ids = {"prompt_logprob_token_ids": candidates}
resp = await client.post(
GEN_ENDPOINT,
json={**payload, "sampling_params": stream_only_ids, "stream": True},
)
assert resp.status_code == 400
assert "prompt_logprob_token_ids" in resp.text
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
resp = await client.post(
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
)
resp.raise_for_status()
empty = np.load(
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
)
assert empty.shape == (0, len(candidates))
@pytest.mark.asyncio
async def test_same_response_as_chat_completions(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
@@ -11,8 +11,10 @@ row), the trailing positions are populated with sentinel values
"""
import numpy as np
import torch
from vllm.logprobs import create_sample_logprobs
from vllm.v1.engine import EngineCoreOutput
from vllm.v1.engine.logprobs import LogprobsProcessor
from vllm.v1.outputs import LogprobsLists
@@ -64,3 +66,24 @@ def test_accepts_exactly_sized_row():
pos = processor.logprobs[0]
assert set(pos.keys()) == {7, 11, 13}
def test_prompt_token_id_logprobs_are_popped_once():
"""DELTA outputs carry fixed-ID prompt scores exactly once."""
processor = _make_processor(num_logprobs=1)
assert processor.pop_prompt_token_id_logprobs() is None
processor.update_from_output(
EngineCoreOutput(
request_id="req-0",
new_token_ids=[],
prompt_token_id_logprobs=torch.tensor(
[[-0.5, -1.5], [-2.5, -3.5]], dtype=torch.float32
),
)
)
scores = processor.pop_prompt_token_id_logprobs()
assert scores is not None
assert scores.tolist() == [[-0.5, -1.5], [-2.5, -3.5]]
assert processor.pop_prompt_token_id_logprobs() is None
+177 -2
View File
@@ -6,7 +6,9 @@ import math
from collections.abc import Generator
from types import SimpleNamespace
from typing import get_args
from unittest.mock import patch
import numpy as np
import pytest
import torch
@@ -23,6 +25,7 @@ from vllm.config.model import LogprobsMode
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
from vllm.platforms import current_platform
from vllm.v1.engine.input_processor import InputProcessor
from ...conftest import HfRunner, VllmRunner
@@ -81,9 +84,9 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]:
yield hf_model
def _model_config(vocab_size: int = 10):
def _model_config(vocab_size: int = 10, max_logprobs: int = 20):
return SimpleNamespace(
max_logprobs=20,
max_logprobs=max_logprobs,
logits_processors=None,
is_diffusion=False,
get_vocab_size=lambda: vocab_size,
@@ -432,6 +435,57 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]):
)
def test_prompt_logprob_token_ids_bounded_by_max_logprobs():
"""The candidate count is bounded by max_logprobs."""
model_config = _model_config(vocab_size=100, max_logprobs=4)
def verify(**kwargs):
SamplingParams(**kwargs).verify(
model_config,
speculative_config=None,
structured_outputs_config=None,
tokenizer=None,
)
verify(prompt_logprob_token_ids=[1, 2, 3, 4])
# The error names the value to raise max_logprobs to.
with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"):
verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5])
# prompt_logprob_start alone is a caller mistake, not a silent no-op.
with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"):
verify(prompt_logprob_start=3)
def test_prompt_logprob_token_ids_require_v2_model_runner():
"""Only the V2 runner scores them; the V1 runner would return None, and
kv-sharing fast prefill would score rows the cross-decoder never ran."""
processor = SimpleNamespace(
model_config=SimpleNamespace(
return_sampling_mask=False, enable_trace_replay=False, is_diffusion=False
),
vllm_config=SimpleNamespace(
reasoning_config=None,
use_v2_model_runner=False,
cache_config=SimpleNamespace(kv_sharing_fast_prefill=False),
),
speculative_config=None,
structured_outputs_config=None,
tokenizer=None,
validate_logits_processors_params=lambda params: None,
)
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
with patch.object(SamplingParams, "verify"):
with pytest.raises(VLLMValidationError, match="V2 model runner"):
InputProcessor._validate_params(processor, params, ("generate",))
processor.vllm_config.use_v2_model_runner = True
InputProcessor._validate_params(processor, params, ("generate",))
processor.vllm_config.cache_config.kv_sharing_fast_prefill = True
with pytest.raises(VLLMValidationError, match="fast-prefill"):
InputProcessor._validate_params(processor, params, ("generate",))
def test_none_logprobs(vllm_model, example_prompts):
"""Engine should return `logprobs` and `prompt_logprobs` as `None`.
@@ -1298,6 +1352,127 @@ def test_prompt_logprobs_with_chunking_and_preemption():
print(f"Test passed with {preemptions} preemptions")
def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
"""Fixed-ID scores stay row-aligned across chunked prefill and preemption."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = [
"The following numbers of the sequence "
+ ", ".join(str(i) for i in range(10))
+ " are:",
"In one word, the capital of France is ",
] + [f"Tell me about the number {i}: " for i in range(32)]
start = 2
candidate_ids = [10, 100, 1000, 10000]
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=40,
min_tokens=20,
prompt_logprob_token_ids=candidate_ids,
prompt_logprob_start=start,
)
with VllmRunner(
"Qwen/Qwen3-0.6B",
max_model_len=512,
enable_chunked_prefill=True,
max_num_batched_tokens=48, # Force prefill chunking
num_gpu_blocks_override=33, # Force preemptions (32 usable + null block)
disable_log_stats=False,
gpu_memory_utilization=0.25,
) as vllm_model:
metrics_before = vllm_model.llm.get_metrics()
outputs = vllm_model.llm.generate(prompts, sampling_params)
for i, output in enumerate(outputs):
scores = output.prompt_token_id_logprobs
assert scores is not None, f"Output {i} missing fixed-ID scores"
expected_shape = (
len(output.prompt_token_ids) - 1 - start,
len(candidate_ids),
)
assert scores.shape == expected_shape, (
f"Output {i} scored {scores.shape}, expected {expected_shape}"
)
assert math.isfinite(float(scores.min()))
assert float(scores.max()) <= 1e-3, "logprobs must be <= 0"
metrics_after = vllm_model.llm.get_metrics()
preemptions_before = next(
(m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0
)
preemptions_after = next(
(m.value for m in metrics_after if m.name == "vllm:num_preemptions"), 0
)
assert preemptions_after - preemptions_before > 0, (
"Test did not trigger any preemptions"
)
# Row alignment is numerical: chunked and preempted scores must match an
# unchunked, unpreempted run of the same requests. Batch composition moves
# bf16 tail logprobs by a few percent; a misaligned row differs by nats.
with VllmRunner(
"Qwen/Qwen3-0.6B",
max_model_len=512,
max_logprobs=64,
gpu_memory_utilization=0.25,
) as reference:
reference_outputs = reference.llm.generate(prompts, sampling_params)
own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs]
self_scored = reference.llm.generate(
prompts,
[
SamplingParams(
max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids
)
for ids in own_ids
],
)
for output, ref in zip(outputs, reference_outputs):
np.testing.assert_allclose(
output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1
)
for output, ids in zip(self_scored, own_ids):
scores = output.prompt_token_id_logprobs
for row, target in enumerate(output.prompt_token_ids[1:]):
expected = output.prompt_logprobs[row + 1][target].logprob
assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3)
def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
"""A prefill that starts past the first scored row returns no scores.
Such a prefill (here a prefix-cache hit the caller opted back into) never
computes the leading rows, so emitting the buffer would return values the
model never produced for them.
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompt = "The capital of France is Paris. " * 20
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=1,
prompt_logprob_token_ids=[10, 100, 1000],
skip_reading_prefix_cache=False,
)
with VllmRunner(
"Qwen/Qwen3-0.6B",
max_model_len=512,
enable_prefix_caching=True,
gpu_memory_utilization=0.25,
) as vllm_model:
first = vllm_model.llm.generate([prompt], sampling_params)[0]
assert first.prompt_token_id_logprobs is not None
assert math.isfinite(float(first.prompt_token_id_logprobs.min()))
# The re-send is served from the prefix cache, so its prefill starts
# past row 0 and the leading rows are never computed.
second = vllm_model.llm.generate([prompt], sampling_params)[0]
assert second.prompt_token_id_logprobs is None
@large_gpu_mark(min_gb=24)
def test_token_logprobs_large_batch_int64_row_offset():
"""Regression: logprob kernel row offset (row * vocab_size) must use int64.
@@ -41,6 +41,14 @@ def test_sampling_params_trace_field_rejects_empty_list():
)
def test_sampling_params_trace_field_rejects_prompt_scoring():
params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3])
with pytest.raises(ValueError, match="prompt_logprob_token_ids"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
)
def test_sampling_params_trace_field_requires_single_output():
params = SamplingParams(n=2, trace_decode_token_ids=[1])
with pytest.raises(ValueError, match="requires n=1"):