[Frontend] Introduce Renderer for processing chat messages (using ModelConfig) (#30200)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+14
-54
@@ -37,10 +37,6 @@ from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
apply_hf_chat_template,
|
||||
apply_mistral_chat_template,
|
||||
parse_chat_messages,
|
||||
resolve_chat_template_content_format,
|
||||
)
|
||||
from vllm.entrypoints.pooling.score.utils import (
|
||||
ScoreContentPartParam,
|
||||
@@ -786,7 +782,7 @@ class LLM:
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[TokensPrompt]:
|
||||
) -> list[TextPrompt | TokensPrompt]:
|
||||
"""
|
||||
Generate prompt for a chat conversation. The pre-processed
|
||||
prompt can then be used as input for the other LLM methods.
|
||||
@@ -807,63 +803,27 @@ class LLM:
|
||||
# messages is list[...]
|
||||
list_of_messages = [cast(list[ChatCompletionMessageParam], messages)]
|
||||
|
||||
tokenizer = self.get_tokenizer()
|
||||
model_config = self.model_config
|
||||
resolved_content_format = resolve_chat_template_content_format(
|
||||
chat_template,
|
||||
tools,
|
||||
chat_template_content_format,
|
||||
tokenizer,
|
||||
model_config=model_config,
|
||||
)
|
||||
renderer = self.llm_engine.renderer
|
||||
|
||||
_chat_template_kwargs: dict[str, Any] = dict(
|
||||
chat_template=chat_template,
|
||||
add_generation_prompt=add_generation_prompt,
|
||||
continue_final_message=continue_final_message,
|
||||
tools=tools,
|
||||
)
|
||||
_chat_template_kwargs.update(chat_template_kwargs or {})
|
||||
chat_template_kwargs = {
|
||||
"chat_template": chat_template,
|
||||
"add_generation_prompt": add_generation_prompt,
|
||||
"continue_final_message": continue_final_message,
|
||||
"tools": tools,
|
||||
**(chat_template_kwargs or {}),
|
||||
}
|
||||
|
||||
prompts: list[TokensPrompt] = []
|
||||
prompts = list[TextPrompt | TokensPrompt]()
|
||||
|
||||
for msgs in list_of_messages:
|
||||
# NOTE: _parse_chat_message_content_parts() currently doesn't
|
||||
# NOTE: renderer.render_messages() currently doesn't
|
||||
# handle mm_processor_kwargs, since there is no implementation in
|
||||
# the chat message parsing for it.
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
_, prompt = renderer.render_messages(
|
||||
msgs,
|
||||
model_config,
|
||||
content_format=resolved_content_format,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
if isinstance(tokenizer, MistralTokenizer):
|
||||
prompt_token_ids = apply_mistral_chat_template(
|
||||
tokenizer,
|
||||
messages=msgs,
|
||||
**_chat_template_kwargs,
|
||||
)
|
||||
else:
|
||||
prompt_str = apply_hf_chat_template(
|
||||
tokenizer=tokenizer,
|
||||
conversation=conversation,
|
||||
model_config=model_config,
|
||||
**_chat_template_kwargs,
|
||||
)
|
||||
# Special tokens are already included in chat templates so
|
||||
# should not be added by the tokenizer in this case.
|
||||
prompt_token_ids = tokenizer.encode(
|
||||
prompt_str, add_special_tokens=False
|
||||
)
|
||||
|
||||
prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
|
||||
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
if mm_processor_kwargs is not None:
|
||||
prompt["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
|
||||
|
||||
Reference in New Issue
Block a user