[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
+68
-1
@@ -289,6 +289,12 @@ class SamplingParams(
|
||||
prompt_logprobs: int | None = None
|
||||
"""Number of log probabilities to return per prompt token.
|
||||
When set to -1, return all `vocab_size` log probabilities."""
|
||||
prompt_logprob_token_ids: list[int] | None = None
|
||||
"""Token IDs to score at each scored causal prompt row, where row i scores
|
||||
them as predictions of prompt token i + 1. The last prompt row is excluded,
|
||||
so the result has `max(prompt_len - 1 - prompt_logprob_start, 0)` rows."""
|
||||
prompt_logprob_start: int | None = None
|
||||
"""First causal prompt row to score; defaults to the first row."""
|
||||
logprob_token_ids: list[int] | None = None
|
||||
"""Specific token IDs to return logprobs for. More efficient than
|
||||
logprobs=-1 when you only need logprobs for a small set of tokens.
|
||||
@@ -398,6 +404,8 @@ class SamplingParams(
|
||||
min_tokens: int = 0,
|
||||
logprobs: int | None = None,
|
||||
prompt_logprobs: int | None = None,
|
||||
prompt_logprob_token_ids: list[int] | None = None,
|
||||
prompt_logprob_start: int | None = None,
|
||||
detokenize: bool = True,
|
||||
skip_special_tokens: bool = True,
|
||||
spaces_between_special_tokens: bool = True,
|
||||
@@ -464,6 +472,8 @@ class SamplingParams(
|
||||
min_tokens=min_tokens,
|
||||
logprobs=logprobs,
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
prompt_logprob_token_ids=prompt_logprob_token_ids,
|
||||
prompt_logprob_start=prompt_logprob_start,
|
||||
logprob_token_ids=logprob_token_ids,
|
||||
detokenize=detokenize,
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
@@ -550,7 +560,10 @@ class SamplingParams(
|
||||
# If prefix caching is enabled,
|
||||
# the output of prompt logprobs may less than n_prompt_tokens,
|
||||
# we need to skip reading cache at this request.
|
||||
self.skip_reading_prefix_cache = self.prompt_logprobs is not None
|
||||
self.skip_reading_prefix_cache = (
|
||||
self.prompt_logprobs is not None
|
||||
or self.prompt_logprob_token_ids is not None
|
||||
)
|
||||
|
||||
def _verify_args(self) -> None:
|
||||
_verify_num_sequences(self.n, "n")
|
||||
@@ -902,6 +915,56 @@ class SamplingParams(
|
||||
value=num_prompt_logprobs,
|
||||
)
|
||||
|
||||
# Validate prompt_logprob_token_ids.
|
||||
if self.prompt_logprob_token_ids is not None:
|
||||
n = len(self.prompt_logprob_token_ids)
|
||||
if n == 0:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids must not be empty.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=n,
|
||||
)
|
||||
if n > max_logprobs:
|
||||
raise VLLMValidationError(
|
||||
f"Requested prompt_logprob_token_ids of length {n}, "
|
||||
f"which is greater than max allowed: {max_logprobs}. "
|
||||
f"Set max_logprobs (--max-logprobs) to at least {n}.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=n,
|
||||
)
|
||||
vocab_size = model_config.get_vocab_size()
|
||||
invalid_token_ids = [
|
||||
token_id
|
||||
for token_id in self.prompt_logprob_token_ids
|
||||
if token_id < 0 or token_id >= vocab_size
|
||||
]
|
||||
if invalid_token_ids:
|
||||
raise VLLMValidationError(
|
||||
f"token_id(s) {invalid_token_ids} in "
|
||||
f"prompt_logprob_token_ids contain out-of-vocab token ids. "
|
||||
f"Vocabulary size: {vocab_size}",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=invalid_token_ids,
|
||||
)
|
||||
if len(set(self.prompt_logprob_token_ids)) != n:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_token_ids must not contain duplicates.",
|
||||
parameter="prompt_logprob_token_ids",
|
||||
value=self.prompt_logprob_token_ids,
|
||||
)
|
||||
if self.prompt_logprob_start is not None and self.prompt_logprob_start < 0:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_start must be non-negative.",
|
||||
parameter="prompt_logprob_start",
|
||||
value=self.prompt_logprob_start,
|
||||
)
|
||||
elif self.prompt_logprob_start is not None:
|
||||
raise VLLMValidationError(
|
||||
"prompt_logprob_start requires prompt_logprob_token_ids.",
|
||||
parameter="prompt_logprob_start",
|
||||
value=self.prompt_logprob_start,
|
||||
)
|
||||
|
||||
def _validate_stop_token_ids(self, model_config: ModelConfig) -> None:
|
||||
"""Validate stop_token_ids are within vocabulary range."""
|
||||
if not self.stop_token_ids:
|
||||
@@ -969,6 +1032,10 @@ class SamplingParams(
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with prompt_logprobs."
|
||||
)
|
||||
if self.prompt_logprob_token_ids is not None:
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with prompt_logprob_token_ids."
|
||||
)
|
||||
if speculative_config is not None:
|
||||
raise ValueError(
|
||||
"trace_decode_token_ids is not supported with speculative decoding."
|
||||
|
||||
Reference in New Issue
Block a user