[Bugfix] Set SamplingParams.max_tokens for OpenAI requests if not provided by user (#6954)
This commit is contained in:
@@ -11,7 +11,7 @@ from typing_extensions import Annotated
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.entrypoints.openai.logits_processors import get_logits_processors
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sampling_params import LogitsProcessor, SamplingParams
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
|
||||
@@ -215,15 +215,22 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
|
||||
# doc: end-chat-completion-extra-params
|
||||
|
||||
def to_sampling_params(self,
|
||||
tokenizer: PreTrainedTokenizer) -> SamplingParams:
|
||||
# We now allow logprobs being true without top_logrobs.
|
||||
def to_sampling_params(
|
||||
self, tokenizer: PreTrainedTokenizer,
|
||||
guided_decode_logits_processor: Optional[LogitsProcessor],
|
||||
default_max_tokens: int) -> SamplingParams:
|
||||
max_tokens = self.max_tokens
|
||||
if max_tokens is None:
|
||||
max_tokens = default_max_tokens
|
||||
|
||||
# We now allow logprobs being true without top_logrobs.
|
||||
logits_processors = get_logits_processors(
|
||||
logit_bias=self.logit_bias,
|
||||
allowed_token_ids=None,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
if guided_decode_logits_processor:
|
||||
logits_processors.append(guided_decode_logits_processor)
|
||||
|
||||
return SamplingParams(
|
||||
n=self.n,
|
||||
@@ -241,7 +248,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
logprobs=self.top_logprobs if self.logprobs else None,
|
||||
prompt_logprobs=self.top_logprobs if self.echo else None,
|
||||
ignore_eos=self.ignore_eos,
|
||||
max_tokens=self.max_tokens,
|
||||
max_tokens=max_tokens,
|
||||
min_tokens=self.min_tokens,
|
||||
use_beam_search=self.use_beam_search,
|
||||
early_stopping=self.early_stopping,
|
||||
@@ -395,7 +402,14 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
|
||||
# doc: end-completion-extra-params
|
||||
|
||||
def to_sampling_params(self, tokenizer: PreTrainedTokenizer):
|
||||
def to_sampling_params(
|
||||
self, tokenizer: PreTrainedTokenizer,
|
||||
guided_decode_logits_processor: Optional[LogitsProcessor],
|
||||
default_max_tokens: int) -> SamplingParams:
|
||||
max_tokens = self.max_tokens
|
||||
if max_tokens is None:
|
||||
max_tokens = default_max_tokens
|
||||
|
||||
echo_without_generation = self.echo and self.max_tokens == 0
|
||||
|
||||
logits_processors = get_logits_processors(
|
||||
@@ -403,6 +417,8 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
allowed_token_ids=self.allowed_token_ids,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
if guided_decode_logits_processor:
|
||||
logits_processors.append(guided_decode_logits_processor)
|
||||
|
||||
return SamplingParams(
|
||||
n=self.n,
|
||||
@@ -419,7 +435,7 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
stop_token_ids=self.stop_token_ids,
|
||||
logprobs=self.logprobs,
|
||||
ignore_eos=self.ignore_eos,
|
||||
max_tokens=self.max_tokens if not echo_without_generation else 1,
|
||||
max_tokens=max_tokens if not echo_without_generation else 1,
|
||||
min_tokens=self.min_tokens,
|
||||
use_beam_search=self.use_beam_search,
|
||||
early_stopping=self.early_stopping,
|
||||
|
||||
Reference in New Issue
Block a user