[Bugfix] Propagate default stop_token_ids to per-request SamplingParams (#35076)
Signed-off-by: sriganesh123 <arjulasriganesh@gmail.com>
This commit is contained in:
@@ -610,6 +610,18 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
|
||||
)
|
||||
|
||||
# Merge server-default stop_token_ids (e.g., model-specific tokens
|
||||
# like </call> for gpt-oss) with any request-specified ones
|
||||
stop_token_ids = self.stop_token_ids
|
||||
default_stop_ids = default_sampling_params.get("stop_token_ids")
|
||||
if default_stop_ids:
|
||||
if not stop_token_ids:
|
||||
stop_token_ids = list(default_stop_ids)
|
||||
else:
|
||||
stop_token_ids = list(
|
||||
dict.fromkeys([*stop_token_ids, *default_stop_ids])
|
||||
)
|
||||
|
||||
prompt_logprobs = self.prompt_logprobs
|
||||
if prompt_logprobs is None and self.echo:
|
||||
prompt_logprobs = self.top_logprobs
|
||||
@@ -661,7 +673,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
min_p=min_p,
|
||||
seed=self.seed,
|
||||
stop=self.stop,
|
||||
stop_token_ids=self.stop_token_ids,
|
||||
stop_token_ids=stop_token_ids,
|
||||
logprobs=self.top_logprobs if self.logprobs else None,
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
ignore_eos=self.ignore_eos,
|
||||
|
||||
@@ -288,6 +288,18 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
"min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
|
||||
)
|
||||
|
||||
# Merge server-default stop_token_ids (e.g., model-specific tokens
|
||||
# like </call> for gpt-oss) with any request-specified ones
|
||||
stop_token_ids = self.stop_token_ids
|
||||
default_stop_ids = default_sampling_params.get("stop_token_ids")
|
||||
if default_stop_ids:
|
||||
if not stop_token_ids:
|
||||
stop_token_ids = list(default_stop_ids)
|
||||
else:
|
||||
stop_token_ids = list(
|
||||
dict.fromkeys([*stop_token_ids, *default_stop_ids])
|
||||
)
|
||||
|
||||
prompt_logprobs = self.prompt_logprobs
|
||||
if prompt_logprobs is None and self.echo:
|
||||
prompt_logprobs = self.logprobs
|
||||
@@ -341,7 +353,7 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
min_p=min_p,
|
||||
seed=self.seed,
|
||||
stop=self.stop,
|
||||
stop_token_ids=self.stop_token_ids,
|
||||
stop_token_ids=stop_token_ids,
|
||||
logprobs=self.logprobs,
|
||||
ignore_eos=self.ignore_eos,
|
||||
max_tokens=max_tokens if not echo_without_generation else 1,
|
||||
|
||||
Reference in New Issue
Block a user