[Frontend] Split ServingRender into renderer and entrypoint. (#44285)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -32,7 +32,7 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
@@ -55,6 +55,8 @@ from vllm.entrypoints.serve.utils.server_utils import (
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tracing import instrument
|
||||
@@ -360,10 +362,24 @@ async def init_app_state(
|
||||
)
|
||||
await state.openai_serving_models.init_static_loras()
|
||||
|
||||
state.openai_serving_render = OpenAIServingRender(
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
model_registry=state.openai_serving_models.registry,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
@@ -378,13 +394,19 @@ async def init_app_state(
|
||||
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
state.openai_serving_models,
|
||||
state.openai_serving_render,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
state.serving_render = ServingRender(
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
state.online_derenderer,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||
@@ -423,8 +445,8 @@ async def init_render_app_state(
|
||||
"""
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
served_model_names = args.served_model_name or [args.model]
|
||||
model_registry = OpenAIModelRegistry(
|
||||
@@ -443,10 +465,24 @@ async def init_render_app_state(
|
||||
renderer = renderer_from_config(vllm_config)
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
state.openai_serving_render = OpenAIServingRender(
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
model_registry=model_registry,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
@@ -462,13 +498,19 @@ async def init_render_app_state(
|
||||
state.openai_serving_models = model_registry
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
model_registry,
|
||||
state.openai_serving_render,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
state.serving_render = ServingRender(
|
||||
model_registry,
|
||||
state.online_renderer,
|
||||
state.online_derenderer,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
state.vllm_config = vllm_config
|
||||
# Disable stats logging — there is no engine to poll.
|
||||
|
||||
Reference in New Issue
Block a user