Fix/async chat serving (#2727)
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import codecs
|
||||
import time
|
||||
from typing import (AsyncGenerator, AsyncIterator, Awaitable, Iterable, List,
|
||||
@@ -40,9 +41,11 @@ class OpenAIServingChat(OpenAIServing):
|
||||
chat_template: Optional[str] = None):
|
||||
super().__init__(engine=engine,
|
||||
served_model_names=served_model_names,
|
||||
lora_modules=lora_modules)
|
||||
lora_modules=lora_modules,
|
||||
await_post_init=self._load_chat_template(
|
||||
chat_template=chat_template))
|
||||
|
||||
self.response_role = response_role
|
||||
self._load_chat_template(chat_template)
|
||||
|
||||
def _parse_chat_message_content(
|
||||
self,
|
||||
@@ -356,7 +359,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
return response
|
||||
|
||||
def _load_chat_template(self, chat_template: Optional[str]):
|
||||
async def _load_chat_template(self, chat_template: Optional[str]):
|
||||
while self.tokenizer is None:
|
||||
# Give the parent class time to load the tokenizer
|
||||
await asyncio.sleep(0.1)
|
||||
tokenizer = self.tokenizer
|
||||
|
||||
if chat_template is not None:
|
||||
|
||||
@@ -2,7 +2,7 @@ import asyncio
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Awaitable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from pydantic import Field
|
||||
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
|
||||
@@ -29,8 +29,11 @@ class LoRAModulePath:
|
||||
|
||||
class OpenAIServing:
|
||||
|
||||
def __init__(self, engine: AsyncLLMEngine, served_model_names: List[str],
|
||||
lora_modules: Optional[List[LoRAModulePath]]):
|
||||
def __init__(self,
|
||||
engine: AsyncLLMEngine,
|
||||
served_model_names: List[str],
|
||||
lora_modules: Optional[List[LoRAModulePath]],
|
||||
await_post_init: Optional[Awaitable[Any]] = None):
|
||||
self.engine = engine
|
||||
self.served_model_names = served_model_names
|
||||
if lora_modules is None:
|
||||
@@ -56,12 +59,12 @@ class OpenAIServing:
|
||||
if event_loop is not None and event_loop.is_running():
|
||||
# If the current is instanced by Ray Serve,
|
||||
# there is already a running event loop
|
||||
event_loop.create_task(self._post_init())
|
||||
event_loop.create_task(self._post_init(await_post_init))
|
||||
else:
|
||||
# When using single vLLM without engine_use_ray
|
||||
asyncio.run(self._post_init())
|
||||
asyncio.run(self._post_init(await_post_init))
|
||||
|
||||
async def _post_init(self):
|
||||
async def _post_init(self, await_post_init):
|
||||
engine_model_config = await self.engine.get_model_config()
|
||||
self.max_model_len = engine_model_config.max_model_len
|
||||
|
||||
@@ -73,6 +76,9 @@ class OpenAIServing:
|
||||
trust_remote_code=engine_model_config.trust_remote_code,
|
||||
truncation_side="left")
|
||||
|
||||
if await_post_init is not None:
|
||||
await await_post_init
|
||||
|
||||
async def show_available_models(self) -> ModelList:
|
||||
"""Show available models. Right now we only have one model."""
|
||||
model_cards = [
|
||||
|
||||
Reference in New Issue
Block a user