Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
bc21cba967
commit
e319f86f15
+54
-32
@@ -11,7 +11,8 @@ from functools import cached_property
|
||||
from typing import Annotated, Any
|
||||
|
||||
import msgspec
|
||||
from pydantic import BeforeValidator
|
||||
import numpy as np
|
||||
from pydantic import BeforeValidator, GetPydanticSchema, StrictInt
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -20,6 +21,7 @@ from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from vllm.utils.torch_utils import make_ndarray_with_pad
|
||||
from vllm.v1.serial_utils import PydanticMsgspecMixin
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -289,10 +291,14 @@ class SamplingParams(
|
||||
prompt_logprobs: int | None = None
|
||||
"""Number of log probabilities to return per prompt token.
|
||||
When set to -1, return all `vocab_size` log probabilities."""
|
||||
prompt_logprob_token_ids: list[int] | None = None
|
||||
"""Token IDs to score at each scored causal prompt row, where row i scores
|
||||
them as predictions of prompt token i + 1. The last prompt row is excluded,
|
||||
so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows."""
|
||||
prompt_logprob_token_ids: (
|
||||
Annotated[np.ndarray, GetPydanticSchema(lambda _, h: h(list[list[StrictInt]]))]
|
||||
| None
|
||||
) = None
|
||||
"""Token IDs to score per causal prompt row: an integer array of shape
|
||||
[num_rows, num_ids] or nested lists; -1 entries (padding) score -inf.
|
||||
Row i scores its IDs as predictions of prompt token
|
||||
prompt_logprob_start + i + 1, so the last prompt row is excluded."""
|
||||
prompt_logprob_start: int | None = None
|
||||
"""First causal prompt row to score; defaults to the first row."""
|
||||
logprob_token_ids: list[int] | None = None
|
||||
@@ -404,7 +410,7 @@ class SamplingParams(
|
||||
min_tokens: int = 0,
|
||||
logprobs: int | None = None,
|
||||
prompt_logprobs: int | None = None,
|
||||
prompt_logprob_token_ids: list[int] | None = None,
|
||||
prompt_logprob_token_ids: np.ndarray | list[list[int]] | None = None,
|
||||
prompt_logprob_start: int | None = None,
|
||||
detokenize: bool = True,
|
||||
skip_special_tokens: bool = True,
|
||||
@@ -916,42 +922,58 @@ class SamplingParams(
|
||||
)
|
||||
|
||||
# Validate prompt_logprob_token_ids.
|
||||
if self.prompt_logprob_token_ids is not None:
|
||||
n = len(self.prompt_logprob_token_ids)
|
||||
if n == 0:
|
||||
ids = self.prompt_logprob_token_ids
|
||||
if ids is not None:
|
||||
shape: tuple[int, ...] = ()
|
||||
if isinstance(ids, list) and all(isinstance(row, list) for row in ids):
|
||||
shape = (len(ids), max(map(len, ids), default=0))
|
||||
elif isinstance(ids, np.ndarray) and ids.dtype.kind in "iu":
|
||||
shape = ids.shape
|
||||
if len(shape) != 2 or 0 in shape:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids must not be empty.",
|
||||
"prompt_logprob_token_ids must be a non-empty integer array "
|
||||
"of shape [num_rows, num_ids].",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=n,
|
||||
value=getattr(ids, "shape", type(ids).__name__),
|
||||
)
|
||||
if n > max_logprobs:
|
||||
max_rows = model_config.max_model_len - 1
|
||||
if shape[0] > max_rows:
|
||||
raise VLLMValidationError(
|
||||
f"Requested prompt_logprob_token_ids of length {n}, "
|
||||
f"which is greater than max allowed: {max_logprobs}. "
|
||||
f"Set max_logprobs (--max-logprobs) to at least {n}.",
|
||||
f"prompt_logprob_token_ids has {shape[0]} rows, but a prompt "
|
||||
f"has at most {max_rows} scored rows (max_model_len - 1).",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=n,
|
||||
value=shape[0],
|
||||
)
|
||||
num_ids = shape[1]
|
||||
if num_ids > max_logprobs:
|
||||
raise VLLMValidationError(
|
||||
f"Requested {num_ids} token ids per row in "
|
||||
f"prompt_logprob_token_ids, which is greater than max allowed: "
|
||||
f"{max_logprobs}. "
|
||||
f"Set max_logprobs (--max-logprobs) to at least {num_ids}.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=num_ids,
|
||||
)
|
||||
if isinstance(ids, list):
|
||||
# Pad only after the checks above bound the allocation.
|
||||
try:
|
||||
ids = make_ndarray_with_pad(ids, -1, np.int64, max_len=num_ids)
|
||||
except (TypeError, ValueError, OverflowError) as e:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids must contain integer token ids.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
) from e
|
||||
vocab_size = model_config.get_vocab_size()
|
||||
invalid_token_ids = [
|
||||
token_id
|
||||
for token_id in self.prompt_logprob_token_ids
|
||||
if token_id < 0 or token_id >= vocab_size
|
||||
]
|
||||
if invalid_token_ids:
|
||||
lo, hi = int(ids.min()), int(ids.max())
|
||||
if lo < -1 or hi >= vocab_size:
|
||||
raise VLLMValidationError(
|
||||
f"token_id(s) {invalid_token_ids} in "
|
||||
f"prompt_logprob_token_ids contain out-of-vocab token ids. "
|
||||
f"Vocabulary size: {vocab_size}",
|
||||
"prompt_logprob_token_ids contain out-of-vocab token ids "
|
||||
f"(-1 pads a row). Vocabulary size: {vocab_size}",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=invalid_token_ids,
|
||||
)
|
||||
if len(set(self.prompt_logprob_token_ids)) != n:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids must not contain duplicates.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=self.prompt_logprob_token_ids,
|
||||
value=[lo, hi],
|
||||
)
|
||||
# In-vocab IDs fit int32, like the runner's other token ID buffers.
|
||||
self.prompt_logprob_token_ids = np.ascontiguousarray(ids, dtype=np.int32)
|
||||
if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_start must be non-negative.",
|
||||
|
||||
Reference in New Issue
Block a user