Support logit bias for OpenAI API (#3027)

This commit is contained in:
Dylan Hawk
2024-02-27 11:51:53 +08:00
committed by GitHub
parent 4bd18ec0c7
commit e0ade06d63
4 changed files with 83 additions and 12 deletions
@@ -264,10 +264,9 @@ class OpenAIServingCompletion(OpenAIServing):
See https://platform.openai.com/docs/api-reference/completions/create
for the API specification. This API mimics the OpenAI Completion API.
NOTE: Currently we do not support the following features:
NOTE: Currently we do not support the following feature:
- suffix (the language models we currently support do not support
suffix)
- logit_bias (to be supported by vLLM engine)
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
@@ -277,9 +276,6 @@ class OpenAIServingCompletion(OpenAIServing):
if request.suffix is not None:
return self.create_error_response(
"suffix is not currently supported")
if request.logit_bias is not None and len(request.logit_bias) > 0:
return self.create_error_response(
"logit_bias is not currently supported")
model_name = request.model
request_id = f"cmpl-{random_uuid()}"