[Refactor]Reduce duplicate code in serving_chat (#26627)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -63,7 +63,7 @@ from vllm.entrypoints.openai.protocol import (
|
||||
TranslationRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.tool_parsers import ToolParser
|
||||
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
|
||||
from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig
|
||||
from vllm.entrypoints.utils import _validate_truncation_size
|
||||
from vllm.inputs.data import PromptType
|
||||
@@ -82,6 +82,7 @@ from vllm.multimodal import ( # noqa: F401 - Required to resolve Pydantic error
|
||||
)
|
||||
from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tracing import (
|
||||
contains_trace_headers,
|
||||
@@ -274,6 +275,50 @@ class OpenAIServing:
|
||||
self.model_config = self.models.model_config
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
|
||||
def _get_tool_parser(
|
||||
self, tool_parser_name: Optional[str] = None, enable_auto_tools: bool = False
|
||||
) -> Optional[Callable[[AnyTokenizer], ToolParser]]:
|
||||
"""Get the tool parser based on the name."""
|
||||
parser = None
|
||||
if not enable_auto_tools or tool_parser_name is None:
|
||||
return parser
|
||||
logger.info(
|
||||
'"auto" tool choice has been enabled please note that while'
|
||||
" the parallel_tool_calls client option is preset for "
|
||||
"compatibility reasons, it will be ignored."
|
||||
)
|
||||
|
||||
try:
|
||||
if tool_parser_name == "pythonic" and self.model_config.model.startswith(
|
||||
"meta-llama/Llama-3.2"
|
||||
):
|
||||
logger.warning(
|
||||
"Llama3.2 models may struggle to emit valid pythonic tool calls"
|
||||
)
|
||||
parser = ToolParserManager.get_tool_parser(tool_parser_name)
|
||||
except Exception as e:
|
||||
raise TypeError(
|
||||
"Error: --enable-auto-tool-choice requires "
|
||||
f"tool_parser:'{tool_parser_name}' which has not "
|
||||
"been registered"
|
||||
) from e
|
||||
return parser
|
||||
|
||||
def _get_reasoning_parser(
|
||||
self,
|
||||
reasoning_parser_name: str,
|
||||
) -> Optional[Callable[[AnyTokenizer], ReasoningParser]]:
|
||||
"""Get the reasoning parser based on the name."""
|
||||
parser = None
|
||||
if not reasoning_parser_name:
|
||||
return None
|
||||
try:
|
||||
parser = ReasoningParserManager.get_reasoning_parser(reasoning_parser_name)
|
||||
assert parser is not None
|
||||
except Exception as e:
|
||||
raise TypeError(f"{reasoning_parser_name=} has not been registered") from e
|
||||
return parser
|
||||
|
||||
async def reset_mm_cache(self) -> None:
|
||||
self.processor.clear_mm_cache()
|
||||
await self.engine_client.reset_mm_cache()
|
||||
|
||||
Reference in New Issue
Block a user