[Frontend] Gracefully handle missing chat template and fix CI failure (#7238)
Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
co-authored by
Roger Wang
parent
7b261092de
commit
66d617e343
@@ -1,8 +1,9 @@
|
||||
import codecs
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import (Awaitable, Iterable, List, Optional, Tuple, Union, cast,
|
||||
final)
|
||||
from pathlib import Path
|
||||
from typing import (Any, Awaitable, Iterable, List, Optional, Tuple, Union,
|
||||
cast, final)
|
||||
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
@@ -22,6 +23,7 @@ from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MultiModalDataDict
|
||||
from vllm.multimodal.utils import async_get_and_parse_image
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -69,13 +71,17 @@ class ChatMessageParseResult:
|
||||
mm_futures: List[Awaitable[MultiModalDataDict]]
|
||||
|
||||
|
||||
def load_chat_template(chat_template: Optional[str]) -> Optional[str]:
|
||||
def load_chat_template(
|
||||
chat_template: Optional[Union[Path, str]]) -> Optional[str]:
|
||||
if chat_template is None:
|
||||
return None
|
||||
try:
|
||||
with open(chat_template, "r") as f:
|
||||
resolved_chat_template = f.read()
|
||||
except OSError as e:
|
||||
if isinstance(chat_template, Path):
|
||||
raise
|
||||
|
||||
JINJA_CHARS = "{}\n"
|
||||
if not any(c in chat_template for c in JINJA_CHARS):
|
||||
msg = (f"The supplied chat template ({chat_template}) "
|
||||
@@ -208,3 +214,28 @@ def parse_chat_messages(
|
||||
mm_futures.extend(parse_result.mm_futures)
|
||||
|
||||
return conversation, mm_futures
|
||||
|
||||
|
||||
def apply_chat_template(
|
||||
tokenizer: AnyTokenizer,
|
||||
conversation: List[ConversationMessage],
|
||||
chat_template: Optional[str],
|
||||
*,
|
||||
tokenize: bool = False, # Different from HF's default
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
if chat_template is None and tokenizer.chat_template is None:
|
||||
raise ValueError(
|
||||
"As of transformers v4.44, default chat template is no longer "
|
||||
"allowed, so you must provide a chat template if the tokenizer "
|
||||
"does not define one.")
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
chat_template=chat_template,
|
||||
tokenize=tokenize,
|
||||
**kwargs,
|
||||
)
|
||||
assert isinstance(prompt, str)
|
||||
|
||||
return prompt
|
||||
|
||||
@@ -190,8 +190,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
default=None,
|
||||
description=(
|
||||
"A Jinja template to use for this conversion. "
|
||||
"If this is not passed, the model's default chat template will be "
|
||||
"used instead."),
|
||||
"As of transformers v4.44, default chat template is no longer "
|
||||
"allowed, so you must provide a chat template if the tokenizer "
|
||||
"does not define one."),
|
||||
)
|
||||
chat_template_kwargs: Optional[Dict[str, Any]] = Field(
|
||||
default=None,
|
||||
|
||||
@@ -10,6 +10,7 @@ from transformers import PreTrainedTokenizer
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import AsyncEngineClient
|
||||
from vllm.entrypoints.chat_utils import (ConversationMessage,
|
||||
apply_chat_template,
|
||||
load_chat_template,
|
||||
parse_chat_messages)
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
@@ -99,16 +100,15 @@ class OpenAIServingChat(OpenAIServing):
|
||||
tool.model_dump() for tool in request.tools
|
||||
]
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
prompt = apply_chat_template(
|
||||
tokenizer,
|
||||
conversation=conversation,
|
||||
tokenize=False,
|
||||
chat_template=request.chat_template or self.chat_template,
|
||||
add_generation_prompt=request.add_generation_prompt,
|
||||
tools=tool_dicts,
|
||||
documents=request.documents,
|
||||
chat_template=request.chat_template or self.chat_template,
|
||||
**(request.chat_template_kwargs or {}),
|
||||
)
|
||||
assert isinstance(prompt, str)
|
||||
except Exception as e:
|
||||
logger.error("Error in applying chat template from request: %s", e)
|
||||
return self.create_error_response(str(e))
|
||||
|
||||
@@ -2,7 +2,9 @@ from typing import List, Optional, Union
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import AsyncEngineClient
|
||||
from vllm.entrypoints.chat_utils import load_chat_template, parse_chat_messages
|
||||
from vllm.entrypoints.chat_utils import (apply_chat_template,
|
||||
load_chat_template,
|
||||
parse_chat_messages)
|
||||
from vllm.entrypoints.logger import RequestLogger
|
||||
# yapf conflicts with isort for this block
|
||||
# yapf: disable
|
||||
@@ -70,12 +72,12 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
logger.warning(
|
||||
"Multi-modal inputs are ignored during tokenization")
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
add_generation_prompt=request.add_generation_prompt,
|
||||
prompt = apply_chat_template(
|
||||
tokenizer,
|
||||
conversation=conversation,
|
||||
tokenize=False,
|
||||
chat_template=self.chat_template)
|
||||
assert isinstance(prompt, str)
|
||||
chat_template=self.chat_template,
|
||||
add_generation_prompt=request.add_generation_prompt,
|
||||
)
|
||||
else:
|
||||
prompt = request.prompt
|
||||
|
||||
|
||||
Reference in New Issue
Block a user