[Feature] Per-row candidate IDs for prefill token scoring (M2 of #56860) (#56984)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-03 14:17:09 +00:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent bc21cba967
commit e319f86f15
17 changed files with 366 additions and 162 deletions
@@ -318,11 +318,11 @@ async def test_generate_logprobs(client, logprobs_value):
@pytest.mark.asyncio
async def test_generate_prompt_token_id_logprobs(client):
token_ids = [11, 22, 33, 44, 55]
candidates = [22, 33, 44, 55]
rows = [[33], [44, 100], [55]]
sampling_params = {
"max_tokens": 1,
"prompt_logprobs": 0,
"prompt_logprob_token_ids": candidates,
"prompt_logprob_token_ids": rows,
"prompt_logprob_start": 1,
}
payload = {
@@ -335,14 +335,13 @@ async def test_generate_prompt_token_id_logprobs(client):
data = resp.json()
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
assert scores.shape == (len(token_ids) - 2, len(candidates))
assert scores.shape == (3, 2)
for row, target in enumerate(token_ids[2:]):
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
assert scores[row, candidates.index(target)] == pytest.approx(
expected, abs=1e-3
)
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
assert np.isneginf(scores[[0, 2], 1]).all()
stream_only_ids = {"prompt_logprob_token_ids": candidates}
stream_only_ids = {"prompt_logprob_token_ids": rows}
resp = await client.post(
GEN_ENDPOINT,
json={**payload, "sampling_params": stream_only_ids, "stream": True},
@@ -350,15 +349,12 @@ async def test_generate_prompt_token_id_logprobs(client):
assert resp.status_code == 400
assert "prompt_logprob_token_ids" in resp.text
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
resp = await client.post(
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
)
resp.raise_for_status()
empty = np.load(
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
)
assert empty.shape == (0, len(candidates))
assert resp.status_code == 400
assert "scored rows" in resp.text
@pytest.mark.asyncio
+115 -40
View File
@@ -11,6 +11,7 @@ from unittest.mock import patch
import numpy as np
import pytest
import torch
from pydantic import TypeAdapter, ValidationError
from tests.utils import large_gpu_mark
from tests.v1.sample.utils import (
@@ -20,7 +21,7 @@ from tests.v1.sample.utils import (
compute_correct_cumulative_logprob,
get_test_batch,
)
from vllm import SamplingParams
from vllm import SamplingParams, TokensPrompt
from vllm.config.model import LogprobsMode
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
@@ -88,6 +89,7 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]:
def _model_config(vocab_size: int = 10, max_logprobs: int = 20):
return SimpleNamespace(
max_logprobs=max_logprobs,
max_model_len=16,
logits_processors=None,
is_diffusion=False,
get_vocab_size=lambda: vocab_size,
@@ -443,24 +445,58 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]):
)
def test_prompt_logprob_token_ids_bounded_by_max_logprobs():
"""The candidate count is bounded by max_logprobs."""
def test_prompt_logprob_token_ids_validation():
"""Rows may be ragged (-1 padded); the width is bounded by max_logprobs."""
model_config = _model_config(vocab_size=100, max_logprobs=4)
def verify(**kwargs):
SamplingParams(**kwargs).verify(
params = SamplingParams(**kwargs)
params.verify(
model_config,
speculative_config=None,
structured_outputs_config=None,
tokenizer=None,
)
return params.prompt_logprob_token_ids
verify(prompt_logprob_token_ids=[1, 2, 3, 4])
assert verify(prompt_logprob_token_ids=[[], [1], [4, 3, 2, 1]]).tolist() == [
[-1] * 4,
[1, -1, -1, -1],
[4, 3, 2, 1],
]
# Any integer layout is normalized to what the worker uploads:
# C-contiguous native int32.
odd = np.asfortranarray(np.array([[1, 2], [3, -1]], dtype=">i8"))
ids = verify(prompt_logprob_token_ids=odd)
assert ids.dtype == np.int32 and ids.flags.c_contiguous
assert ids.tolist() == [[1, 2], [3, -1]]
# The pydantic path (token-in/token-out API, /docs) takes strict integer lists.
adapter = TypeAdapter(SamplingParams)
for bad in ("[[1.5]]", '[["3"]]', "[[true]]", "[1, 2]"):
with pytest.raises(ValidationError):
adapter.validate_json(f'{{"prompt_logprob_token_ids": {bad}}}')
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
verify(prompt_logprob_token_ids=np.array([1, 2]))
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
verify(prompt_logprob_token_ids=[1, 2])
for bad_ids in ([[None]], [[2**63]], [[[1]]], [[[1], 2]]):
with pytest.raises(VLLMValidationError, match="integer token ids"):
verify(prompt_logprob_token_ids=bad_ids)
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
verify(prompt_logprob_token_ids=np.array([[2**64 - 1]], dtype=np.uint64))
# The error names the value to raise max_logprobs to.
with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"):
verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5])
verify(prompt_logprob_token_ids=[[1], [1, 2, 3, 4, 5]])
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
verify(prompt_logprob_token_ids=[[1], [100]])
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
verify(prompt_logprob_token_ids=[[1], [-2]])
with pytest.raises(VLLMValidationError, match="non-empty"):
verify(prompt_logprob_token_ids=[[], []])
# Rows are bounded by max_model_len before the table is padded.
with pytest.raises(VLLMValidationError, match="at most 15 scored rows"):
verify(prompt_logprob_token_ids=[[]] * 15 + [[1]])
# prompt_logprob_start alone is a caller mistake, not a silent no-op.
with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"):
verify(prompt_logprob_start=3)
@@ -483,7 +519,7 @@ def test_prompt_logprob_token_ids_require_v2_model_runner():
tokenizer=None,
validate_logits_processors_params=lambda params: None,
)
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
params = SamplingParams(prompt_logprob_token_ids=[[1, 2]])
with patch.object(SamplingParams, "verify"):
with pytest.raises(VLLMValidationError, match="V2 model runner"):
InputProcessor._validate_params(processor, params, ("generate",))
@@ -1369,7 +1405,7 @@ def test_prompt_logprobs_with_chunking_and_preemption():
def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
"""Fixed-ID scores stay row-aligned across chunked prefill and preemption."""
"""Per-row scores stay row-aligned across chunked prefill and preemption."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = [
@@ -1380,14 +1416,9 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
] + [f"Tell me about the number {i}: " for i in range(32)]
start = 2
candidate_ids = [10, 100, 1000, 10000]
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=40,
min_tokens=20,
prompt_logprob_token_ids=candidate_ids,
prompt_logprob_start=start,
)
def candidate_ids(j):
return [10 + j, 100 + j, 1000 + j, 10000 + j]
with VllmRunner(
"Qwen/Qwen3-0.6B",
@@ -1398,15 +1429,38 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
disable_log_stats=False,
gpu_memory_utilization=0.25,
) as vllm_model:
tokenizer = vllm_model.llm.get_tokenizer()
token_prompts = [
TokensPrompt(prompt_token_ids=tokenizer(p).input_ids) for p in prompts
]
def make_params(row_ids):
return [
SamplingParams(
temperature=0.0,
max_tokens=40,
min_tokens=20,
prompt_logprob_token_ids=[
row_ids(j)
for j in range(len(p["prompt_token_ids"]) - 1 - start)
],
prompt_logprob_start=start,
)
for p in token_prompts
]
metrics_before = vllm_model.llm.get_metrics()
outputs = vllm_model.llm.generate(prompts, sampling_params)
full_params = make_params(candidate_ids)
for params in full_params: # the array form, as a top-k tensor would be
params.prompt_logprob_token_ids = np.array(params.prompt_logprob_token_ids)
outputs = vllm_model.llm.generate(token_prompts, full_params)
for i, output in enumerate(outputs):
scores = output.prompt_token_id_logprobs
assert scores is not None, f"Output {i} missing fixed-ID scores"
assert scores is not None, f"Output {i} missing per-row scores"
expected_shape = (
len(output.prompt_token_ids) - 1 - start,
len(candidate_ids),
4,
)
assert scores.shape == expected_shape, (
f"Output {i} scored {scores.shape}, expected {expected_shape}"
@@ -1414,6 +1468,23 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
assert math.isfinite(float(scores.min()))
assert float(scores.max()) <= 1e-3, "logprobs must be <= 0"
# Ragged rows: each row keeps a prefix of the candidates and must
# reproduce that prefix of the full scores, with -inf padding.
ragged_params = make_params(lambda j: candidate_ids(j)[: 1 + j % 4])
ragged_outputs = vllm_model.llm.generate(token_prompts, ragged_params)
for output, ragged_output in zip(outputs, ragged_outputs):
expected = np.full_like(output.prompt_token_id_logprobs, -np.inf)
for j in range(len(expected)):
n = 1 + j % 4
expected[j, :n] = output.prompt_token_id_logprobs[j, :n]
np.testing.assert_allclose(
ragged_output.prompt_token_id_logprobs, expected, atol=1e-2
)
# The row count is checked against the prompt at admission.
with pytest.raises(VLLMValidationError, match="scored rows"):
vllm_model.llm.generate(token_prompts[0], ragged_params[1])
metrics_after = vllm_model.llm.get_metrics()
preemptions_before = next(
(m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0
@@ -1429,31 +1500,32 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
# unchunked, unpreempted run of the same requests. Batch composition moves
# bf16 tail logprobs by a few percent; a misaligned row differs by nats.
with VllmRunner(
"Qwen/Qwen3-0.6B",
max_model_len=512,
max_logprobs=64,
gpu_memory_utilization=0.25,
"Qwen/Qwen3-0.6B", max_model_len=512, gpu_memory_utilization=0.25
) as reference:
reference_outputs = reference.llm.generate(prompts, sampling_params)
own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs]
reference_outputs = reference.llm.generate(token_prompts, full_params)
self_scored = reference.llm.generate(
prompts,
token_prompts,
[
SamplingParams(
max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids
max_tokens=1,
prompt_logprobs=0,
prompt_logprob_token_ids=[[t] for t in p["prompt_token_ids"][1:]],
)
for ids in own_ids
for p in token_prompts
],
)
for output, ref in zip(outputs, reference_outputs):
np.testing.assert_allclose(
output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1
output.prompt_token_id_logprobs,
ref.prompt_token_id_logprobs,
rtol=0.1,
atol=0.2,
)
for output, ids in zip(self_scored, own_ids):
scores = output.prompt_token_id_logprobs
for output in self_scored:
scores = output.prompt_token_id_logprobs[:, 0]
for row, target in enumerate(output.prompt_token_ids[1:]):
expected = output.prompt_logprobs[row + 1][target].logprob
assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3)
assert scores[row] == pytest.approx(expected, abs=1e-3)
def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
@@ -1465,13 +1537,7 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
"""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompt = "The capital of France is Paris. " * 20
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=1,
prompt_logprob_token_ids=[10, 100, 1000],
skip_reading_prefix_cache=False,
)
text = "The capital of France is Paris. " * 20
with VllmRunner(
"Qwen/Qwen3-0.6B",
@@ -1479,6 +1545,15 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
enable_prefix_caching=True,
gpu_memory_utilization=0.25,
) as vllm_model:
token_ids = vllm_model.llm.get_tokenizer()(text).input_ids
prompt = TokensPrompt(prompt_token_ids=token_ids)
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=1,
prompt_logprob_token_ids=np.tile([10, 100, 1000], (len(token_ids) - 1, 1)),
skip_reading_prefix_cache=False,
)
first = vllm_model.llm.generate([prompt], sampling_params)[0]
assert first.prompt_token_id_logprobs is not None
assert math.isfinite(float(first.prompt_token_id_logprobs.min()))
+3 -1
View File
@@ -42,7 +42,9 @@ def test_sampling_params_trace_field_rejects_empty_list():
def test_sampling_params_trace_field_rejects_prompt_scoring():
params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3])
params = SamplingParams(
trace_decode_token_ids=[1], prompt_logprob_token_ids=[[2, 3]]
)
with pytest.raises(ValueError, match="prompt_logprob_token_ids"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None