[Models] Kimi-K2.5 (#33131)
Signed-off-by: wanglinian <wanglinian@stu.pku.edu.cn> Signed-off-by: wangln19 <96399074+wangln19@users.noreply.github.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: wanglinian <wanglinian@stu.pku.edu.cn> Co-authored-by: wangln19 <96399074+wangln19@users.noreply.github.com> Co-authored-by: Zaida Zhou <58739961+zhouzaida@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
wanglinian
wangln19
Zaida Zhou
Isotr0py
Nick Hill
youkaichao
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
6c00645712
commit
b539f988e1
@@ -20,9 +20,11 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
ChatTemplateResolutionError,
|
||||
ConversationMessage,
|
||||
build_video_prompts_from_mm_data,
|
||||
load_chat_template,
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
rebuild_mm_uuids_from_mm_data,
|
||||
)
|
||||
from vllm.inputs import TextPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
@@ -547,6 +549,40 @@ class HfRenderer(RendererLike):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
# model which uses unified vision chunks for both images and videos.
|
||||
if (
|
||||
getattr(model_config.hf_config, "use_unified_vision_chunk", False)
|
||||
and mm_uuids is not None
|
||||
and mm_data is not None
|
||||
):
|
||||
mm_uuids = rebuild_mm_uuids_from_mm_data(mm_uuids, mm_data)
|
||||
|
||||
# get video placehoder, replace it with runtime video-chunk prompts
|
||||
video_placeholder = getattr(
|
||||
model_config.hf_config, "video_placeholder", None
|
||||
)
|
||||
if video_placeholder and isinstance(prompt_raw, str):
|
||||
video_prompts = build_video_prompts_from_mm_data(mm_data)
|
||||
|
||||
# replace in order
|
||||
prompt_raw_parts = prompt_raw.split(video_placeholder)
|
||||
if len(prompt_raw_parts) == len(video_prompts) + 1:
|
||||
prompt_raw = "".join(
|
||||
[
|
||||
prompt_raw_parts[i] + video_prompts[i]
|
||||
for i in range(len(video_prompts))
|
||||
]
|
||||
)
|
||||
prompt_raw += prompt_raw_parts[-1]
|
||||
else:
|
||||
logger.warning(
|
||||
"Number of video placeholders (%d) does not match "
|
||||
"number of videos (%d) in the request.",
|
||||
len(prompt_raw_parts) - 1,
|
||||
len(video_prompts),
|
||||
)
|
||||
|
||||
prompt = (
|
||||
TextPrompt(prompt=prompt_raw)
|
||||
if isinstance(prompt_raw, str)
|
||||
@@ -587,6 +623,40 @@ class HfRenderer(RendererLike):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
# model which uses unified vision chunks for both images and videos.
|
||||
if (
|
||||
getattr(model_config.hf_config, "use_unified_vision_chunk", False)
|
||||
and mm_uuids is not None
|
||||
and mm_data is not None
|
||||
):
|
||||
mm_uuids = rebuild_mm_uuids_from_mm_data(mm_uuids, mm_data)
|
||||
|
||||
# get video placehoder, replace it with runtime video-chunk prompts
|
||||
video_placeholder = getattr(
|
||||
model_config.hf_config, "video_placeholder", None
|
||||
)
|
||||
if video_placeholder and isinstance(prompt_raw, str):
|
||||
video_prompts = build_video_prompts_from_mm_data(mm_data)
|
||||
|
||||
# replace in order
|
||||
prompt_raw_parts = prompt_raw.split(video_placeholder)
|
||||
if len(prompt_raw_parts) == len(video_prompts) + 1:
|
||||
prompt_raw = "".join(
|
||||
[
|
||||
prompt_raw_parts[i] + video_prompts[i]
|
||||
for i in range(len(video_prompts))
|
||||
]
|
||||
)
|
||||
prompt_raw += prompt_raw_parts[-1]
|
||||
else:
|
||||
logger.warning(
|
||||
"Number of video placeholders (%d) does not match "
|
||||
"number of videos (%d) in the request.",
|
||||
len(prompt_raw_parts) - 1,
|
||||
len(video_prompts),
|
||||
)
|
||||
|
||||
prompt = (
|
||||
TextPrompt(prompt=prompt_raw)
|
||||
if isinstance(prompt_raw, str)
|
||||
|
||||
Reference in New Issue
Block a user