Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
bc21cba967
commit
e319f86f15
@@ -318,11 +318,11 @@ async def test_generate_logprobs(client, logprobs_value):
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_prompt_token_id_logprobs(client):
|
||||
token_ids = [11, 22, 33, 44, 55]
|
||||
candidates = [22, 33, 44, 55]
|
||||
rows = [[33], [44, 100], [55]]
|
||||
sampling_params = {
|
||||
"max_tokens": 1,
|
||||
"prompt_logprobs": 0,
|
||||
"prompt_logprob_token_ids": candidates,
|
||||
"prompt_logprob_token_ids": rows,
|
||||
"prompt_logprob_start": 1,
|
||||
}
|
||||
payload = {
|
||||
@@ -335,14 +335,13 @@ async def test_generate_prompt_token_id_logprobs(client):
|
||||
data = resp.json()
|
||||
|
||||
scores = np.load(io.BytesIO(base64.b64decode(data["prompt_token_id_logprobs"])))
|
||||
assert scores.shape == (len(token_ids) - 2, len(candidates))
|
||||
assert scores.shape == (3, 2)
|
||||
for row, target in enumerate(token_ids[2:]):
|
||||
expected = data["prompt_logprobs"][row + 2][str(target)]["logprob"]
|
||||
assert scores[row, candidates.index(target)] == pytest.approx(
|
||||
expected, abs=1e-3
|
||||
)
|
||||
assert scores[row, 0] == pytest.approx(expected, abs=1e-3)
|
||||
assert np.isneginf(scores[[0, 2], 1]).all()
|
||||
|
||||
stream_only_ids = {"prompt_logprob_token_ids": candidates}
|
||||
stream_only_ids = {"prompt_logprob_token_ids": rows}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={**payload, "sampling_params": stream_only_ids, "stream": True},
|
||||
@@ -350,15 +349,12 @@ async def test_generate_prompt_token_id_logprobs(client):
|
||||
assert resp.status_code == 400
|
||||
assert "prompt_logprob_token_ids" in resp.text
|
||||
|
||||
past_end = {**sampling_params, "prompt_logprob_start": len(token_ids)}
|
||||
too_few_rows = {**sampling_params, "prompt_logprob_token_ids": rows[:2]}
|
||||
resp = await client.post(
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": past_end}
|
||||
GEN_ENDPOINT, json={**payload, "sampling_params": too_few_rows}
|
||||
)
|
||||
resp.raise_for_status()
|
||||
empty = np.load(
|
||||
io.BytesIO(base64.b64decode(resp.json()["prompt_token_id_logprobs"]))
|
||||
)
|
||||
assert empty.shape == (0, len(candidates))
|
||||
assert resp.status_code == 400
|
||||
assert "scored rows" in resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -11,6 +11,7 @@ from unittest.mock import patch
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from tests.utils import large_gpu_mark
|
||||
from tests.v1.sample.utils import (
|
||||
@@ -20,7 +21,7 @@ from tests.v1.sample.utils import (
|
||||
compute_correct_cumulative_logprob,
|
||||
get_test_batch,
|
||||
)
|
||||
from vllm import SamplingParams
|
||||
from vllm import SamplingParams, TokensPrompt
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
@@ -88,6 +89,7 @@ def hf_model(hf_runner) -> Generator[HfRunner, None, None]:
|
||||
def _model_config(vocab_size: int = 10, max_logprobs: int = 20):
|
||||
return SimpleNamespace(
|
||||
max_logprobs=max_logprobs,
|
||||
max_model_len=16,
|
||||
logits_processors=None,
|
||||
is_diffusion=False,
|
||||
get_vocab_size=lambda: vocab_size,
|
||||
@@ -443,24 +445,58 @@ def test_logprob_token_ids_validate_vocab_bounds_invalid(token_ids: list[int]):
|
||||
)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_bounded_by_max_logprobs():
|
||||
"""The candidate count is bounded by max_logprobs."""
|
||||
def test_prompt_logprob_token_ids_validation():
|
||||
"""Rows may be ragged (-1 padded); the width is bounded by max_logprobs."""
|
||||
model_config = _model_config(vocab_size=100, max_logprobs=4)
|
||||
|
||||
def verify(**kwargs):
|
||||
SamplingParams(**kwargs).verify(
|
||||
params = SamplingParams(**kwargs)
|
||||
params.verify(
|
||||
model_config,
|
||||
speculative_config=None,
|
||||
structured_outputs_config=None,
|
||||
tokenizer=None,
|
||||
)
|
||||
return params.prompt_logprob_token_ids
|
||||
|
||||
verify(prompt_logprob_token_ids=[1, 2, 3, 4])
|
||||
assert verify(prompt_logprob_token_ids=[[], [1], [4, 3, 2, 1]]).tolist() == [
|
||||
[-1] * 4,
|
||||
[1, -1, -1, -1],
|
||||
[4, 3, 2, 1],
|
||||
]
|
||||
# Any integer layout is normalized to what the worker uploads:
|
||||
# C-contiguous native int32.
|
||||
odd = np.asfortranarray(np.array([[1, 2], [3, -1]], dtype=">i8"))
|
||||
ids = verify(prompt_logprob_token_ids=odd)
|
||||
assert ids.dtype == np.int32 and ids.flags.c_contiguous
|
||||
assert ids.tolist() == [[1, 2], [3, -1]]
|
||||
# The pydantic path (token-in/token-out API, /docs) takes strict integer lists.
|
||||
adapter = TypeAdapter(SamplingParams)
|
||||
for bad in ("[[1.5]]", '[["3"]]', "[[true]]", "[1, 2]"):
|
||||
with pytest.raises(ValidationError):
|
||||
adapter.validate_json(f'{{"prompt_logprob_token_ids": {bad}}}')
|
||||
|
||||
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
|
||||
verify(prompt_logprob_token_ids=np.array([1, 2]))
|
||||
with pytest.raises(VLLMValidationError, match=r"shape \[num_rows, num_ids\]"):
|
||||
verify(prompt_logprob_token_ids=[1, 2])
|
||||
for bad_ids in ([[None]], [[2**63]], [[[1]]], [[[1], 2]]):
|
||||
with pytest.raises(VLLMValidationError, match="integer token ids"):
|
||||
verify(prompt_logprob_token_ids=bad_ids)
|
||||
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
|
||||
verify(prompt_logprob_token_ids=np.array([[2**64 - 1]], dtype=np.uint64))
|
||||
# The error names the value to raise max_logprobs to.
|
||||
with pytest.raises(VLLMValidationError, match=r"max_logprobs.*at least 5"):
|
||||
verify(prompt_logprob_token_ids=[1, 2, 3, 4, 5])
|
||||
|
||||
verify(prompt_logprob_token_ids=[[1], [1, 2, 3, 4, 5]])
|
||||
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
|
||||
verify(prompt_logprob_token_ids=[[1], [100]])
|
||||
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
|
||||
verify(prompt_logprob_token_ids=[[1], [-2]])
|
||||
with pytest.raises(VLLMValidationError, match="non-empty"):
|
||||
verify(prompt_logprob_token_ids=[[], []])
|
||||
# Rows are bounded by max_model_len before the table is padded.
|
||||
with pytest.raises(VLLMValidationError, match="at most 15 scored rows"):
|
||||
verify(prompt_logprob_token_ids=[[]] * 15 + [[1]])
|
||||
# prompt_logprob_start alone is a caller mistake, not a silent no-op.
|
||||
with pytest.raises(VLLMValidationError, match="requires prompt_logprob_token_ids"):
|
||||
verify(prompt_logprob_start=3)
|
||||
@@ -483,7 +519,7 @@ def test_prompt_logprob_token_ids_require_v2_model_runner():
|
||||
tokenizer=None,
|
||||
validate_logits_processors_params=lambda params: None,
|
||||
)
|
||||
params = SamplingParams(prompt_logprob_token_ids=[1, 2])
|
||||
params = SamplingParams(prompt_logprob_token_ids=[[1, 2]])
|
||||
with patch.object(SamplingParams, "verify"):
|
||||
with pytest.raises(VLLMValidationError, match="V2 model runner"):
|
||||
InputProcessor._validate_params(processor, params, ("generate",))
|
||||
@@ -1369,7 +1405,7 @@ def test_prompt_logprobs_with_chunking_and_preemption():
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
"""Fixed-ID scores stay row-aligned across chunked prefill and preemption."""
|
||||
"""Per-row scores stay row-aligned across chunked prefill and preemption."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
prompts = [
|
||||
@@ -1380,14 +1416,9 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
] + [f"Tell me about the number {i}: " for i in range(32)]
|
||||
|
||||
start = 2
|
||||
candidate_ids = [10, 100, 1000, 10000]
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=40,
|
||||
min_tokens=20,
|
||||
prompt_logprob_token_ids=candidate_ids,
|
||||
prompt_logprob_start=start,
|
||||
)
|
||||
|
||||
def candidate_ids(j):
|
||||
return [10 + j, 100 + j, 1000 + j, 10000 + j]
|
||||
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
@@ -1398,15 +1429,38 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
disable_log_stats=False,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
tokenizer = vllm_model.llm.get_tokenizer()
|
||||
token_prompts = [
|
||||
TokensPrompt(prompt_token_ids=tokenizer(p).input_ids) for p in prompts
|
||||
]
|
||||
|
||||
def make_params(row_ids):
|
||||
return [
|
||||
SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=40,
|
||||
min_tokens=20,
|
||||
prompt_logprob_token_ids=[
|
||||
row_ids(j)
|
||||
for j in range(len(p["prompt_token_ids"]) - 1 - start)
|
||||
],
|
||||
prompt_logprob_start=start,
|
||||
)
|
||||
for p in token_prompts
|
||||
]
|
||||
|
||||
metrics_before = vllm_model.llm.get_metrics()
|
||||
outputs = vllm_model.llm.generate(prompts, sampling_params)
|
||||
full_params = make_params(candidate_ids)
|
||||
for params in full_params: # the array form, as a top-k tensor would be
|
||||
params.prompt_logprob_token_ids = np.array(params.prompt_logprob_token_ids)
|
||||
outputs = vllm_model.llm.generate(token_prompts, full_params)
|
||||
|
||||
for i, output in enumerate(outputs):
|
||||
scores = output.prompt_token_id_logprobs
|
||||
assert scores is not None, f"Output {i} missing fixed-ID scores"
|
||||
assert scores is not None, f"Output {i} missing per-row scores"
|
||||
expected_shape = (
|
||||
len(output.prompt_token_ids) - 1 - start,
|
||||
len(candidate_ids),
|
||||
4,
|
||||
)
|
||||
assert scores.shape == expected_shape, (
|
||||
f"Output {i} scored {scores.shape}, expected {expected_shape}"
|
||||
@@ -1414,6 +1468,23 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
assert math.isfinite(float(scores.min()))
|
||||
assert float(scores.max()) <= 1e-3, "logprobs must be <= 0"
|
||||
|
||||
# Ragged rows: each row keeps a prefix of the candidates and must
|
||||
# reproduce that prefix of the full scores, with -inf padding.
|
||||
ragged_params = make_params(lambda j: candidate_ids(j)[: 1 + j % 4])
|
||||
ragged_outputs = vllm_model.llm.generate(token_prompts, ragged_params)
|
||||
for output, ragged_output in zip(outputs, ragged_outputs):
|
||||
expected = np.full_like(output.prompt_token_id_logprobs, -np.inf)
|
||||
for j in range(len(expected)):
|
||||
n = 1 + j % 4
|
||||
expected[j, :n] = output.prompt_token_id_logprobs[j, :n]
|
||||
np.testing.assert_allclose(
|
||||
ragged_output.prompt_token_id_logprobs, expected, atol=1e-2
|
||||
)
|
||||
|
||||
# The row count is checked against the prompt at admission.
|
||||
with pytest.raises(VLLMValidationError, match="scored rows"):
|
||||
vllm_model.llm.generate(token_prompts[0], ragged_params[1])
|
||||
|
||||
metrics_after = vllm_model.llm.get_metrics()
|
||||
preemptions_before = next(
|
||||
(m.value for m in metrics_before if m.name == "vllm:num_preemptions"), 0
|
||||
@@ -1429,31 +1500,32 @@ def test_prompt_logprob_token_ids_with_chunking_and_preemption(monkeypatch):
|
||||
# unchunked, unpreempted run of the same requests. Batch composition moves
|
||||
# bf16 tail logprobs by a few percent; a misaligned row differs by nats.
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
max_logprobs=64,
|
||||
gpu_memory_utilization=0.25,
|
||||
"Qwen/Qwen3-0.6B", max_model_len=512, gpu_memory_utilization=0.25
|
||||
) as reference:
|
||||
reference_outputs = reference.llm.generate(prompts, sampling_params)
|
||||
own_ids = [list(dict.fromkeys(o.prompt_token_ids)) for o in reference_outputs]
|
||||
reference_outputs = reference.llm.generate(token_prompts, full_params)
|
||||
self_scored = reference.llm.generate(
|
||||
prompts,
|
||||
token_prompts,
|
||||
[
|
||||
SamplingParams(
|
||||
max_tokens=1, prompt_logprobs=0, prompt_logprob_token_ids=ids
|
||||
max_tokens=1,
|
||||
prompt_logprobs=0,
|
||||
prompt_logprob_token_ids=[[t] for t in p["prompt_token_ids"][1:]],
|
||||
)
|
||||
for ids in own_ids
|
||||
for p in token_prompts
|
||||
],
|
||||
)
|
||||
for output, ref in zip(outputs, reference_outputs):
|
||||
np.testing.assert_allclose(
|
||||
output.prompt_token_id_logprobs, ref.prompt_token_id_logprobs, rtol=0.1
|
||||
output.prompt_token_id_logprobs,
|
||||
ref.prompt_token_id_logprobs,
|
||||
rtol=0.1,
|
||||
atol=0.2,
|
||||
)
|
||||
for output, ids in zip(self_scored, own_ids):
|
||||
scores = output.prompt_token_id_logprobs
|
||||
for output in self_scored:
|
||||
scores = output.prompt_token_id_logprobs[:, 0]
|
||||
for row, target in enumerate(output.prompt_token_ids[1:]):
|
||||
expected = output.prompt_logprobs[row + 1][target].logprob
|
||||
assert scores[row, ids.index(target)] == pytest.approx(expected, abs=1e-3)
|
||||
assert scores[row] == pytest.approx(expected, abs=1e-3)
|
||||
|
||||
|
||||
def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
@@ -1465,13 +1537,7 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
prompt = "The capital of France is Paris. " * 20
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=1,
|
||||
prompt_logprob_token_ids=[10, 100, 1000],
|
||||
skip_reading_prefix_cache=False,
|
||||
)
|
||||
text = "The capital of France is Paris. " * 20
|
||||
|
||||
with VllmRunner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
@@ -1479,6 +1545,15 @@ def test_prompt_logprob_token_ids_drop_partially_scored_prefills(monkeypatch):
|
||||
enable_prefix_caching=True,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
token_ids = vllm_model.llm.get_tokenizer()(text).input_ids
|
||||
prompt = TokensPrompt(prompt_token_ids=token_ids)
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=1,
|
||||
prompt_logprob_token_ids=np.tile([10, 100, 1000], (len(token_ids) - 1, 1)),
|
||||
skip_reading_prefix_cache=False,
|
||||
)
|
||||
|
||||
first = vllm_model.llm.generate([prompt], sampling_params)[0]
|
||||
assert first.prompt_token_id_logprobs is not None
|
||||
assert math.isfinite(float(first.prompt_token_id_logprobs.min()))
|
||||
|
||||
@@ -42,7 +42,9 @@ def test_sampling_params_trace_field_rejects_empty_list():
|
||||
|
||||
|
||||
def test_sampling_params_trace_field_rejects_prompt_scoring():
|
||||
params = SamplingParams(trace_decode_token_ids=[1], prompt_logprob_token_ids=[2, 3])
|
||||
params = SamplingParams(
|
||||
trace_decode_token_ids=[1], prompt_logprob_token_ids=[[2, 3]]
|
||||
)
|
||||
with pytest.raises(ValueError, match="prompt_logprob_token_ids"):
|
||||
params._validate_trace_replay(
|
||||
_make_model_config(vocab_size=100), speculative_config=None
|
||||
|
||||
Reference in New Issue
Block a user