[Feature] Per-row candidate IDs for prefill token scoring (M2 of #56860) (#56984)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-03 14:17:09 +00:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent bc21cba967
commit e319f86f15
17 changed files with 366 additions and 162 deletions
+54 -32
View File
@@ -11,7 +11,8 @@ from functools import cached_property
from typing import Annotated, Any
import msgspec
from pydantic import BeforeValidator
import numpy as np
from pydantic import BeforeValidator, GetPydanticSchema, StrictInt
from pydantic.dataclasses import dataclass
import vllm.envs as envs
@@ -20,6 +21,7 @@ from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.utils.mistral import is_mistral_tokenizer
from vllm.utils.torch_utils import make_ndarray_with_pad
from vllm.v1.serial_utils import PydanticMsgspecMixin
logger = init_logger(__name__)
@@ -289,10 +291,14 @@ class SamplingParams(
prompt_logprobs: int | None = None
"""Number of log probabilities to return per prompt token.
When set to -1, return all `vocab_size` log probabilities."""
prompt_logprob_token_ids: list[int] | None = None
"""Token IDs to score at each scored causal prompt row, where row i scores
them as predictions of prompt token i + 1. The last prompt row is excluded,
so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows."""
prompt_logprob_token_ids: (
Annotated[np.ndarray, GetPydanticSchema(lambda _, h: h(list[list[StrictInt]]))]
| None
) = None
"""Token IDs to score per causal prompt row: an integer array of shape
[num_rows, num_ids] or nested lists; -1 entries (padding) score -inf.
Row i scores its IDs as predictions of prompt token
prompt_logprob_start + i + 1, so the last prompt row is excluded."""
prompt_logprob_start: int | None = None
"""First causal prompt row to score; defaults to the first row."""
logprob_token_ids: list[int] | None = None
@@ -404,7 +410,7 @@ class SamplingParams(
min_tokens: int = 0,
logprobs: int | None = None,
prompt_logprobs: int | None = None,
prompt_logprob_token_ids: list[int] | None = None,
prompt_logprob_token_ids: np.ndarray | list[list[int]] | None = None,
prompt_logprob_start: int | None = None,
detokenize: bool = True,
skip_special_tokens: bool = True,
@@ -916,42 +922,58 @@ class SamplingParams(
)
# Validate prompt_logprob_token_ids.
if self.prompt_logprob_token_ids is not None:
n = len(self.prompt_logprob_token_ids)
if n == 0:
ids = self.prompt_logprob_token_ids
if ids is not None:
shape: tuple[int, ...] = ()
if isinstance(ids, list) and all(isinstance(row, list) for row in ids):
shape = (len(ids), max(map(len, ids), default=0))
elif isinstance(ids, np.ndarray) and ids.dtype.kind in "iu":
shape = ids.shape
if len(shape) != 2 or 0 in shape:
raise VLLMValidationError(
"prompt_logprob_token_ids must not be empty.",
"prompt_logprob_token_ids must be a non-empty integer array "
"of shape [num_rows, num_ids].",
parameter="prompt_logprob_token_ids",
value=n,
value=getattr(ids, "shape", type(ids).__name__),
)
if n > max_logprobs:
max_rows = model_config.max_model_len - 1
if shape[0] > max_rows:
raise VLLMValidationError(
f"Requested prompt_logprob_token_ids of length {n}, "
f"which is greater than max allowed: {max_logprobs}. "
f"Set max_logprobs (--max-logprobs) to at least {n}.",
f"prompt_logprob_token_ids has {shape[0]} rows, but a prompt "
f"has at most {max_rows} scored rows (max_model_len - 1).",
parameter="prompt_logprob_token_ids",
value=n,
value=shape[0],
)
num_ids = shape[1]
if num_ids > max_logprobs:
raise VLLMValidationError(
f"Requested {num_ids} token ids per row in "
f"prompt_logprob_token_ids, which is greater than max allowed: "
f"{max_logprobs}. "
f"Set max_logprobs (--max-logprobs) to at least {num_ids}.",
parameter="prompt_logprob_token_ids",
value=num_ids,
)
if isinstance(ids, list):
# Pad only after the checks above bound the allocation.
try:
ids = make_ndarray_with_pad(ids, -1, np.int64, max_len=num_ids)
except (TypeError, ValueError, OverflowError) as e:
raise VLLMValidationError(
"prompt_logprob_token_ids must contain integer token ids.",
parameter="prompt_logprob_token_ids",
) from e
vocab_size = model_config.get_vocab_size()
invalid_token_ids = [
token_id
for token_id in self.prompt_logprob_token_ids
if token_id < 0 or token_id >= vocab_size
]
if invalid_token_ids:
lo, hi = int(ids.min()), int(ids.max())
if lo < -1 or hi >= vocab_size:
raise VLLMValidationError(
f"token_id(s) {invalid_token_ids} in "
f"prompt_logprob_token_ids contain out-of-vocab token ids. "
f"Vocabulary size: {vocab_size}",
"prompt_logprob_token_ids contain out-of-vocab token ids "
f"(-1 pads a row). Vocabulary size: {vocab_size}",
parameter="prompt_logprob_token_ids",
value=invalid_token_ids,
)
if len(set(self.prompt_logprob_token_ids)) != n:
raise VLLMValidationError(
"prompt_logprob_token_ids must not contain duplicates.",
parameter="prompt_logprob_token_ids",
value=self.prompt_logprob_token_ids,
value=[lo, hi],
)
# In-vocab IDs fit int32, like the runner's other token ID buffers.
self.prompt_logprob_token_ids = np.ascontiguousarray(ids, dtype=np.int32)
if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0:
raise VLLMValidationError(
"prompt_logprob_start must be non-negative.",