[OpenAI] Fix tool_choice=required streaming when output has trailing extra data (#31610)
Signed-off-by: maylikenoother <ogedengbemary19@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -13,6 +13,7 @@ import partial_json_parser
|
||||
import regex as re
|
||||
from fastapi import Request
|
||||
from openai_harmony import Message as OpenAIMessage
|
||||
from partial_json_parser.core.options import Allow
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
@@ -76,6 +77,7 @@ from vllm.tokenizers.mistral import (
|
||||
)
|
||||
from vllm.tool_parsers import ToolParser
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
|
||||
from vllm.tool_parsers.utils import partial_json_loads
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
|
||||
|
||||
@@ -511,8 +513,12 @@ class OpenAIServingChat(OpenAIServing):
|
||||
# if the current text is empty, we cannot parse it
|
||||
return None, function_name_returned
|
||||
try:
|
||||
obj = partial_json_parser.loads(current_text)
|
||||
except partial_json_parser.core.exceptions.MalformedJSON:
|
||||
flags = Allow.ALL
|
||||
obj, _ = partial_json_loads(current_text, flags)
|
||||
except (
|
||||
partial_json_parser.core.exceptions.MalformedJSON,
|
||||
json.JSONDecodeError,
|
||||
):
|
||||
logger.debug("not enough tokens to parse into JSON yet")
|
||||
obj = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user