[Feature] Add fixed-token prefill scoring (#54335)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-28 07:50:09 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5
parent a7ff44355c
commit b721a4c709
36 changed files with 768 additions and 8 deletions
+68 -1
View File
@@ -289,6 +289,12 @@ class SamplingParams(
prompt_logprobs: int | None = None
"""Number of log probabilities to return per prompt token.
When set to -1, return all `vocab_size` log probabilities."""
prompt_logprob_token_ids: list[int] | None = None
"""Token IDs to score at each scored causal prompt row, where row i scores
them as predictions of prompt token i + 1. The last prompt row is excluded,
so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows."""
prompt_logprob_start: int | None = None
"""First causal prompt row to score; defaults to the first row."""
logprob_token_ids: list[int] | None = None
"""Specific token IDs to return logprobs for. More efficient than
logprobs=-1 when you only need logprobs for a small set of tokens.
@@ -398,6 +404,8 @@ class SamplingParams(
min_tokens: int = 0,
logprobs: int | None = None,
prompt_logprobs: int | None = None,
prompt_logprob_token_ids: list[int] | None = None,
prompt_logprob_start: int | None = None,
detokenize: bool = True,
skip_special_tokens: bool = True,
spaces_between_special_tokens: bool = True,
@@ -464,6 +472,8 @@ class SamplingParams(
min_tokens=min_tokens,
logprobs=logprobs,
prompt_logprobs=prompt_logprobs,
prompt_logprob_token_ids=prompt_logprob_token_ids,
prompt_logprob_start=prompt_logprob_start,
logprob_token_ids=logprob_token_ids,
detokenize=detokenize,
skip_special_tokens=skip_special_tokens,
@@ -550,7 +560,10 @@ class SamplingParams(
# If prefix caching is enabled,
# the output of prompt logprobs may less than n_prompt_tokens,
# we need to skip reading cache at this request.
self.skip_reading_prefix_cache = self.prompt_logprobs is not None
self.skip_reading_prefix_cache = (
self.prompt_logprobs is not None
or self.prompt_logprob_token_ids is not None
)
def _verify_args(self) -> None:
_verify_num_sequences(self.n, "n")
@@ -902,6 +915,56 @@ class SamplingParams(
value=num_prompt_logprobs,
)
# Validate prompt_logprob_token_ids.
if self.prompt_logprob_token_ids is not None:
n = len(self.prompt_logprob_token_ids)
if n == 0:
raise VLLMValidationError(
"prompt_logprob_token_ids must not be empty.",
parameter="prompt_logprob_token_ids",
value=n,
)
if n > max_logprobs:
raise VLLMValidationError(
f"Requested prompt_logprob_token_ids of length {n}, "
f"which is greater than max allowed: {max_logprobs}. "
f"Set max_logprobs (--max-logprobs) to at least {n}.",
parameter="prompt_logprob_token_ids",
value=n,
)
vocab_size = model_config.get_vocab_size()
invalid_token_ids = [
token_id
for token_id in self.prompt_logprob_token_ids
if token_id < 0 or token_id >= vocab_size
]
if invalid_token_ids:
raise VLLMValidationError(
f"token_id(s) {invalid_token_ids} in "
f"prompt_logprob_token_ids contain out-of-vocab token ids. "
f"Vocabulary size: {vocab_size}",
parameter="prompt_logprob_token_ids",
value=invalid_token_ids,
)
if len(set(self.prompt_logprob_token_ids)) != n:
raise VLLMValidationError(
"prompt_logprob_token_ids must not contain duplicates.",
parameter="prompt_logprob_token_ids",
value=self.prompt_logprob_token_ids,
)
if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0:
raise VLLMValidationError(
"prompt_logprob_start must be non-negative.",
parameter="prompt_logprob_start",
value=self.prompt_logprob_start,
)
elif self.prompt_logprob_start is not None:
raise VLLMValidationError(
"prompt_logprob_start requires prompt_logprob_token_ids.",
parameter="prompt_logprob_start",
value=self.prompt_logprob_start,
)
def _validate_stop_token_ids(self, model_config: ModelConfig) -> None:
"""Validate stop_token_ids are within vocabulary range."""
if not self.stop_token_ids:
@@ -969,6 +1032,10 @@ class SamplingParams(
raise ValueError(
"trace_decode_token_ids is not supported with prompt_logprobs."
)
if self.prompt_logprob_token_ids is not None:
raise ValueError(
"trace_decode_token_ids is not supported with prompt_logprob_token_ids."
)
if speculative_config is not None:
raise ValueError(
"trace_decode_token_ids is not supported with speculative decoding."