Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Patrick von Platen
2024-09-11 14:41:55 -07:00
committed by GitHub
co-authored by Roger Wang
parent 775f00f81e
commit d394787e52
8 changed files with 807 additions and 9 deletions
+2 -1
View File
@@ -148,7 +148,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
return f"<|image_{current_count}|>"
if model_type == "minicpmv":
return "(<image>./</image>)"
if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
if model_type in ("blip-2", "chatglm", "fuyu", "paligemma",
"pixtral"):
# These models do not use image tokens in the prompt
return None
if model_type == "qwen":