[Frontend] Introduce Renderer for processing chat messages (using ModelConfig) (#30200)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -28,21 +28,17 @@ def register_pooling_api_routers(app: FastAPI):
|
||||
async def init_pooling_state(
|
||||
engine_client: "EngineClient", state: "State", args: "Namespace"
|
||||
):
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
from vllm.entrypoints.pooling.classify.serving import ServingClassification
|
||||
from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
|
||||
from vllm.entrypoints.pooling.pooling.serving import OpenAIServingPooling
|
||||
from vllm.entrypoints.pooling.score.serving import ServingScores
|
||||
from vllm.entrypoints.utils import process_chat_template
|
||||
from vllm.tasks import POOLING_TASKS
|
||||
|
||||
supported_tasks = await engine_client.get_supported_tasks()
|
||||
|
||||
vllm_config = engine_client.vllm_config
|
||||
|
||||
resolved_chat_template = await process_chat_template(
|
||||
args.chat_template, engine_client, vllm_config.model_config
|
||||
)
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
if args.enable_log_requests:
|
||||
request_logger = RequestLogger(max_log_len=args.max_log_len)
|
||||
|
||||
@@ -54,8 +54,6 @@ class ClassificationMixin(OpenAIServing):
|
||||
"""
|
||||
ctx = cast(ClassificationServeContext, ctx)
|
||||
try:
|
||||
ctx.tokenizer = await self.engine_client.get_tokenizer()
|
||||
|
||||
request_obj = ctx.request
|
||||
|
||||
if isinstance(request_obj, ClassificationChatRequest):
|
||||
@@ -76,7 +74,7 @@ class ClassificationMixin(OpenAIServing):
|
||||
|
||||
_, engine_prompts = await self._preprocess_chat(
|
||||
cast(ChatCompletionRequest, chat_request),
|
||||
ctx.tokenizer,
|
||||
self.renderer,
|
||||
messages,
|
||||
chat_template=(
|
||||
chat_request.chat_template
|
||||
@@ -104,7 +102,7 @@ class ClassificationMixin(OpenAIServing):
|
||||
ctx.engine_prompts = []
|
||||
return None
|
||||
|
||||
renderer = self._get_renderer(ctx.tokenizer)
|
||||
renderer = self._get_completion_renderer()
|
||||
prompt_input = cast(str | list[str], input_data)
|
||||
ctx.engine_prompts = await renderer.render_prompt(
|
||||
prompt_or_prompts=prompt_input,
|
||||
|
||||
@@ -78,13 +78,10 @@ class EmbeddingMixin(OpenAIServing):
|
||||
try:
|
||||
ctx.lora_request = self._maybe_get_adapters(ctx.request)
|
||||
|
||||
tokenizer = await self.engine_client.get_tokenizer()
|
||||
renderer = self._get_renderer(tokenizer)
|
||||
|
||||
if isinstance(ctx.request, EmbeddingChatRequest):
|
||||
_, ctx.engine_prompts = await self._preprocess_chat(
|
||||
ctx.request,
|
||||
tokenizer,
|
||||
self.renderer,
|
||||
ctx.request.messages,
|
||||
chat_template=ctx.request.chat_template or ctx.chat_template,
|
||||
chat_template_content_format=ctx.chat_template_content_format,
|
||||
@@ -93,6 +90,7 @@ class EmbeddingMixin(OpenAIServing):
|
||||
add_special_tokens=ctx.request.add_special_tokens,
|
||||
)
|
||||
else:
|
||||
renderer = self._get_completion_renderer()
|
||||
ctx.engine_prompts = await renderer.render_prompt(
|
||||
prompt_or_prompts=ctx.request.input,
|
||||
config=self._build_render_config(ctx.request),
|
||||
|
||||
@@ -94,12 +94,6 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
try:
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
|
||||
if self.model_config.skip_tokenizer_init:
|
||||
tokenizer = None
|
||||
else:
|
||||
tokenizer = await self.engine_client.get_tokenizer()
|
||||
renderer = self._get_renderer(tokenizer)
|
||||
|
||||
if getattr(request, "dimensions", None) is not None:
|
||||
return self.create_error_response(
|
||||
"dimensions is currently not supported"
|
||||
@@ -140,7 +134,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
|
||||
_, engine_prompts = await self._preprocess_chat(
|
||||
request,
|
||||
tokenizer,
|
||||
self.renderer,
|
||||
request.messages,
|
||||
chat_template=request.chat_template or self.chat_template,
|
||||
chat_template_content_format=self.chat_template_content_format,
|
||||
@@ -149,6 +143,7 @@ class OpenAIServingPooling(OpenAIServing):
|
||||
add_special_tokens=request.add_special_tokens,
|
||||
)
|
||||
elif isinstance(request, PoolingCompletionRequest):
|
||||
renderer = self._get_completion_renderer()
|
||||
engine_prompts = await renderer.render_prompt(
|
||||
prompt_or_prompts=request.input,
|
||||
config=self._build_render_config(request),
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
@@ -63,6 +64,8 @@ class ServingScores(OpenAIServing):
|
||||
)
|
||||
self.score_template = score_template
|
||||
|
||||
self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
async def _embedding_score(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
@@ -283,8 +286,7 @@ class ServingScores(OpenAIServing):
|
||||
raw_request: Request | None = None,
|
||||
) -> list[PoolingRequestOutput] | ErrorResponse:
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
|
||||
tokenizer = await self.engine_client.get_tokenizer()
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
|
||||
truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)
|
||||
|
||||
|
||||
@@ -16,12 +16,12 @@ from vllm.entrypoints.chat_utils import (
|
||||
MultiModalItemTracker,
|
||||
_ContentPart,
|
||||
_parse_chat_message_content_part,
|
||||
apply_hf_chat_template,
|
||||
)
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.model_executor.models.interfaces import supports_score_template
|
||||
from vllm.multimodal.inputs import MultiModalDataDict
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.renderers.hf import safe_apply_chat_template
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
ScoreContentPartParam: TypeAlias = (
|
||||
@@ -224,15 +224,16 @@ def get_score_prompt(
|
||||
# If that fails because there is no such template,
|
||||
# fall back to the default implementation.
|
||||
try:
|
||||
full_prompt = apply_hf_chat_template(
|
||||
full_prompt = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
[
|
||||
{"role": "query", "content": prompt_1},
|
||||
{"role": "document", "content": prompt_2},
|
||||
],
|
||||
score_template,
|
||||
chat_template=score_template,
|
||||
tools=None,
|
||||
model_config=model_config,
|
||||
tokenize=False,
|
||||
)
|
||||
prompt_inputs = tokenizer(full_prompt, **tokenization_kwargs)
|
||||
except ChatTemplateResolutionError:
|
||||
|
||||
Reference in New Issue
Block a user