fix(config): apply presence_penalty/frequency_penalty from override-generation-config (#50769)

Signed-off-by: Chenglun Hu <chenglunhu@gmail.com>
Signed-off-by: hclsys <chenglunhu@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
hcl
2026-09-24 08:48:18 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Harry Mellor
parent 26f49e336a
commit a7d203efe5
5 changed files with 150 additions and 60 deletions
@@ -215,7 +215,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
# https://platform.openai.com/docs/api-reference/chat/create
messages: list[ChatCompletionMessageParam]
model: str | None = None
frequency_penalty: float | None = 0.0
frequency_penalty: float | None = None
logit_bias: dict[str, float] | None = None
logprobs: bool | None = False
top_logprobs: int | None = 0
@@ -226,7 +226,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
)
max_completion_tokens: int | None = None
n: int | None = 1
presence_penalty: float | None = 0.0
presence_penalty: float | None = None
response_format: AnyResponseFormat | None = None
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: StopParam = []
@@ -624,6 +624,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
# Default sampling parameters for chat completion requests
_DEFAULT_SAMPLING_PARAMS: dict = {
"repetition_penalty": 1.0,
"presence_penalty": 0.0,
"frequency_penalty": 0.0,
"temperature": 1.0,
"top_p": 1.0,
"top_k": 0,
@@ -661,28 +663,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
max_tokens: int,
default_sampling_params: dict,
) -> SamplingParams:
# Default parameters
if (repetition_penalty := self.repetition_penalty) is None:
repetition_penalty = default_sampling_params.get(
"repetition_penalty",
self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
)
if (temperature := self.temperature) is None:
temperature = default_sampling_params.get(
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
)
if (top_p := self.top_p) is None:
top_p = default_sampling_params.get(
"top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
)
if (top_k := self.top_k) is None:
top_k = default_sampling_params.get(
"top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
)
if (min_p := self.min_p) is None:
min_p = default_sampling_params.get(
"min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
# Priority: user -> server default -> OpenAI default
sampling_params = {
name: (
value
if (value := getattr(self, name)) is not None
else default_sampling_params.get(name, default)
)
for name, default in self._DEFAULT_SAMPLING_PARAMS.items()
}
# Merge server-default stop_token_ids (e.g., model-specific tokens
# like </call> for gpt-oss) with any request-specified ones
@@ -709,14 +698,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
extra_args["ec_transfer_params"] = self.ec_transfer_params
return SamplingParams.from_optional(
n=self.n,
presence_penalty=self.presence_penalty,
frequency_penalty=self.frequency_penalty,
repetition_penalty=repetition_penalty,
temperature=temperature,
**sampling_params,
watermarking=self.watermarking,
top_p=top_p,
top_k=top_k,
min_p=min_p,
seed=self.seed,
stop=self.stop,
stop_token_ids=stop_token_ids,