[Bugfix][Frontend] Honor default penalties in batch chat completions (#60307)
Signed-off-by: sricursion <sricursion@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -2,7 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Unit tests for presence_penalty / frequency_penalty resolution from
|
||||
default_sampling_params in ChatCompletionRequest and CompletionRequest.
|
||||
default_sampling_params in ChatCompletionRequest, BatchChatCompletionRequest
|
||||
and CompletionRequest.
|
||||
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/50767:
|
||||
these two penalties defaulted to 0.0 (not None) and to_sampling_params()
|
||||
@@ -12,6 +13,7 @@ discarded on the /v1/chat/completions and /v1/completions endpoints.
|
||||
"""
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
@@ -29,6 +31,14 @@ def _chat(**kwargs):
|
||||
)
|
||||
|
||||
|
||||
def _batch(**kwargs):
|
||||
conversation = [{"role": "user", "content": "hi"}]
|
||||
request = BatchChatCompletionRequest(
|
||||
model="test-model", messages=[conversation], **kwargs
|
||||
)
|
||||
return request.to_chat_completion_request(conversation)
|
||||
|
||||
|
||||
class TestChatCompletionPenaltyDefaults:
|
||||
def test_defaults_applied_when_client_omits(self):
|
||||
"""Server-default penalties are applied when the client sends none."""
|
||||
@@ -49,6 +59,22 @@ class TestChatCompletionPenaltyDefaults:
|
||||
assert sp.frequency_penalty == 0.0
|
||||
|
||||
|
||||
class TestBatchChatCompletionPenaltyDefaults:
|
||||
def test_defaults_applied_when_client_omits(self):
|
||||
"""Server-default penalties reach every conversation of a batch."""
|
||||
sp = _batch().to_sampling_params(100, _DEFAULTS)
|
||||
assert sp.presence_penalty == 1.5
|
||||
assert sp.frequency_penalty == 0.5
|
||||
|
||||
def test_client_value_overrides_default(self):
|
||||
"""An explicit client penalty, even 0.0, wins over the server default."""
|
||||
sp = _batch(presence_penalty=0.0, frequency_penalty=0.2).to_sampling_params(
|
||||
100, _DEFAULTS
|
||||
)
|
||||
assert sp.presence_penalty == 0.0
|
||||
assert sp.frequency_penalty == 0.2
|
||||
|
||||
|
||||
class TestCompletionPenaltyDefaults:
|
||||
def test_defaults_applied_when_client_omits(self):
|
||||
sp = CompletionRequest(model="test-model", prompt="hi").to_sampling_params(
|
||||
|
||||
Reference in New Issue
Block a user