[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
@@ -1,9 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from transformers import AutoTokenizer
|
||||
@@ -311,6 +314,52 @@ async def test_generate_logprobs(client, logprobs_value):
|
||||
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_prompt_token_id_logprobs(client):
|
||||
token_ids = [11, 22, 33, 44, 55]
|
||||
candidates = [22, 33, 44, 55]
|
||||
sampling_params = {
|
||||
"max_tokens": 1,
|
||||
"prompt_logprobs": 0,
|
||||
"prompt_logprob_token_ids": candidates,
|
||||
"prompt_logprob_start": 1,
|
||||
}
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": sampling_params,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
|
||||
assert scores.shape == (len(token_ids) - 2, len(candidates))
|
||||
for row, target in enumerate(token_ids[2:]):
|
||||
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
|
||||
assert scores[row, candidates.index(target)] == pytest.approx(
|
||||
expected, abs=1e-3
|
||||
)
|
||||
|
||||
stream_only_ids = {"prompt_logprob_token_ids": candidates}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={**payload, "sampling_params": stream_only_ids, "stream": True},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "prompt_logprob_token_ids" in resp.text
|
||||
|
||||
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
|
||||
)
|
||||
resp.raise_for_status()
|
||||
empty = np.load(
|
||||
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
|
||||
)
|
||||
assert empty.shape == (0, len(candidates))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_response_as_chat_completions(client, tokenizer, messages):
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
|
||||
@@ -11,8 +11,10 @@ row), the trailing positions are populated with sentinel values
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.logprobs import create_sample_logprobs
|
||||
from vllm.v1.engine import EngineCoreOutput
|
||||
from vllm.v1.engine.logprobs import LogprobsProcessor
|
||||
from vllm.v1.outputs import LogprobsLists
|
||||
|
||||
@@ -64,3 +66,24 @@ def test_accepts_exactly_sized_row():
|
||||
|
||||
pos = processor.logprobs[0]
|
||||
assert set(pos.keys()) == {7, 11, 13}
|
||||
|
||||
|
||||
def test_prompt_token_id_logprobs_are_popped_once():
|
||||
"""DELTA outputs carry fixed-ID prompt scores exactly once."""
|
||||
processor = _make_processor(num_logprobs=1)
|
||||
assert processor.pop_prompt_token_id_logprobs() is None
|
||||
|
||||
processor.update_from_output(
|
||||
EngineCoreOutput(
|
||||
request_id="req-0",
|
||||
new_token_ids=[],
|
||||
prompt_token_id_logprobs=torch.tensor(
|
||||
[[-0.5, -1.5], [-2.5, -3.5]], dtype=torch.float32
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
scores = processor.pop_prompt_token_id_logprobs()
|
||||
assert scores is not None
|
||||
assert scores.tolist() == [[-0.5, -1.5], [-2.5, -3.5]]
|
||||
assert processor.pop_prompt_token_id_logprobs() is None
|
||||
|
||||
@@ -6,7 +6,9 @@ import math
|
||||
from collections.abc import Generator
|
||||
from types import SimpleNamespace
|
||||
from typing import get_args
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -23,6 +25,7 @@ from vllm.config.model import LogprobsMode
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
|
||||
from ...conftest import HfRunner, VllmRunner
|
||||
|
||||
@@ -81,9 +84,9 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]:
|
||||
yield hf_model
|
||||
|
||||
|
||||
def _model_config(vocab_size: int = 10):
|
||||
def _model_config(vocab_size: int = 10, max_logprobs: int = 20):
|
||||
return SimpleNamespace(
|
||||
max_logprobs=20,
|
||||
max_logprobs=max_logprobs,
|
||||
logits_processors=None,
|
||||
is_diffusion=False,
|
||||
get_vocab_size=lambda: vocab_size,
|
||||
@@ -432,6 +435,57 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]):
|
||||
)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_bounded_by_max_logprobs():
|
||||
"""The candidate count is bounded by max_logprobs."""
|
||||
model_config = _model_config(vocab_size=100, max_logprobs=4)
|
||||
|
||||
def verify(**kwargs):
|
||||
SamplingParams(**kwargs).verify(
|
||||
model_config,
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
tokenizer=None,
|
||||
)
|
||||
|
||||
verify(prompt_logprob_token_ids=[1, 2, 3, 4])
|
||||
|
||||
# The error names the value to raise max_logprobs to.
|
||||
with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"):
|
||||
verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5])
|
||||
|
||||
# prompt_logprob_start alone is a caller mistake, not a silent no-op.
|
||||
with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"):
|
||||
verify(prompt_logprob_start=3)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_require_v2_model_runner():
|
||||
"""Only the V2 runner scores them; the V1 runner would return None, and
|
||||
kv-sharing fast prefill would score rows the cross-decoder never ran."""
|
||||
processor = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
return_sampling_mask=False, enable_trace_replay=False, is_diffusion=False
|
||||
),
|
||||
vllm_config=SimpleNamespace(
|
||||
reasoning_config=None,
|
||||
use_v2_model_runner=False,
|
||||
cache_config=SimpleNamespace(kv_sharing_fast_prefill=False),
|
||||
),
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
tokenizer=None,
|
||||
validate_logits_processors_params=lambda params: None,
|
||||
)
|
||||
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
|
||||
with patch.object(SamplingParams, "verify"):
|
||||
with pytest.raises(VLLMValidationError, match="V2 model runner"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
processor.vllm_config.use_v2_model_runner = True
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
processor.vllm_config.cache_config.kv_sharing_fast_prefill = True
|
||||
with pytest.raises(VLLMValidationError, match="fast-prefill"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
|
||||
|
||||
def test_none_logprobs(vllm_model, example_prompts):
|
||||
"""Engine should return `logprobs` and `prompt_logprobs` as `None`.
|
||||
|
||||
@@ -1298,6 +1352,127 @@ def test_prompt_logprobs_with_chunking_and_preemption():
|
||||
print(f"Test passed with {preemptions} preemptions")
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
"""Fixed-ID scores stay row-aligned across chunked prefill and preemption."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
prompts = [
|
||||
"The following numbers of the sequence "
|
||||
+ ", ".join(str(i) for i in range(10))
|
||||
+ " are:",
|
||||
"In one word, the capital of France is ",
|
||||
] + [f"Tell me about the number {i}: " for i in range(32)]
|
||||
|
||||
start = 2
|
||||
candidate_ids = [10, 100, 1000, 10000]
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=40,
|
||||
min_tokens=20,
|
||||
prompt_logprob_token_ids=candidate_ids,
|
||||
prompt_logprob_start=start,
|
||||
)
|
||||
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
enable_chunked_prefill=True,
|
||||
max_num_batched_tokens=48, # Force prefill chunking
|
||||
num_gpu_blocks_override=33, # Force preemptions (32 usable + null block)
|
||||
disable_log_stats=False,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
metrics_before = vllm_model.llm.get_metrics()
|
||||
outputs = vllm_model.llm.generate(prompts, sampling_params)
|
||||
|
||||
for i, output in enumerate(outputs):
|
||||
scores = output.prompt_token_id_logprobs
|
||||
assert scores is not None, f"Output {i} missing fixed-ID scores"
|
||||
expected_shape = (
|
||||
len(output.prompt_token_ids) - 1 - start,
|
||||
len(candidate_ids),
|
||||
)
|
||||
assert scores.shape == expected_shape, (
|
||||
f"Output {i} scored {scores.shape}, expected {expected_shape}"
|
||||
)
|
||||
assert math.isfinite(float(scores.min()))
|
||||
assert float(scores.max()) <= 1e-3, "logprobs must be <= 0"
|
||||
|
||||
metrics_after = vllm_model.llm.get_metrics()
|
||||
preemptions_before = next(
|
||||
(m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0
|
||||
)
|
||||
preemptions_after = next(
|
||||
(m.value for m in metrics_after if m.name == "vllm:num_preemptions"), 0
|
||||
)
|
||||
assert preemptions_after - preemptions_before > 0, (
|
||||
"Test did not trigger any preemptions"
|
||||
)
|
||||
|
||||
# Row alignment is numerical: chunked and preempted scores must match an
|
||||
# unchunked, unpreempted run of the same requests. Batch composition moves
|
||||
# bf16 tail logprobs by a few percent; a misaligned row differs by nats.
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
max_logprobs=64,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as reference:
|
||||
reference_outputs = reference.llm.generate(prompts, sampling_params)
|
||||
own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs]
|
||||
self_scored = reference.llm.generate(
|
||||
prompts,
|
||||
[
|
||||
SamplingParams(
|
||||
max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids
|
||||
)
|
||||
for ids in own_ids
|
||||
],
|
||||
)
|
||||
for output, ref in zip(outputs, reference_outputs):
|
||||
np.testing.assert_allclose(
|
||||
output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1
|
||||
)
|
||||
for output, ids in zip(self_scored, own_ids):
|
||||
scores = output.prompt_token_id_logprobs
|
||||
for row, target in enumerate(output.prompt_token_ids[1:]):
|
||||
expected = output.prompt_logprobs[row + 1][target].logprob
|
||||
assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
"""A prefill that starts past the first scored row returns no scores.
|
||||
|
||||
Such a prefill (here a prefix-cache hit the caller opted back into) never
|
||||
computes the leading rows, so emitting the buffer would return values the
|
||||
model never produced for them.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
prompt = "The capital of France is Paris. " * 20
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=1,
|
||||
prompt_logprob_token_ids=[10, 100, 1000],
|
||||
skip_reading_prefix_cache=False,
|
||||
)
|
||||
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
enable_prefix_caching=True,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
first = vllm_model.llm.generate([prompt], sampling_params)[0]
|
||||
assert first.prompt_token_id_logprobs is not None
|
||||
assert math.isfinite(float(first.prompt_token_id_logprobs.min()))
|
||||
|
||||
# The re-send is served from the prefix cache, so its prefill starts
|
||||
# past row 0 and the leading rows are never computed.
|
||||
second = vllm_model.llm.generate([prompt], sampling_params)[0]
|
||||
assert second.prompt_token_id_logprobs is None
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=24)
|
||||
def test_token_logprobs_large_batch_int64_row_offset():
|
||||
"""Regression: logprob kernel row offset (row * vocab_size) must use int64.
|
||||
|
||||
@@ -41,6 +41,14 @@ def test_sampling_params_trace_field_rejects_empty_list():
|
||||
)
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_rejects_prompt_scoring():
|
||||
params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3])
|
||||
with pytest.raises(ValueError, match="prompt_logprob_token_ids"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
)
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_requires_single_output():
|
||||
params = SamplingParams(n=2, trace_decode_token_ids=[1])
|
||||
with pytest.raises(ValueError, match="requires n=1"):
|
||||
|
||||
Reference in New Issue
Block a user