[Frontend] Introduce Renderer for processing chat messages (using ModelConfig) (#30200)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-01-22 12:44:22 +00:00
committed by GitHub
parent 421012b63a
commit d117a4d1a9
48 changed files with 2141 additions and 1585 deletions
+4 -2
View File
@@ -3,6 +3,7 @@
import asyncio
import time
from collections.abc import AsyncGenerator, Mapping
from concurrent.futures import ThreadPoolExecutor
from typing import Any
from fastapi import Request
@@ -63,6 +64,8 @@ class ServingScores(OpenAIServing):
)
self.score_template = score_template
self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)
async def _embedding_score(
self,
tokenizer: TokenizerLike,
@@ -283,8 +286,7 @@ class ServingScores(OpenAIServing):
raw_request: Request | None = None,
) -> list[PoolingRequestOutput] | ErrorResponse:
lora_request = self._maybe_get_adapters(request)
tokenizer = await self.engine_client.get_tokenizer()
tokenizer = self.renderer.get_tokenizer()
truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)