[vlm] Remove vision language config. (#6089)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
co-authored by
Roger Wang
parent
3c6325f0fc
commit
d9e98f42e4
@@ -121,6 +121,11 @@ class LLM:
|
||||
) -> None:
|
||||
if "disable_log_stats" not in kwargs:
|
||||
kwargs["disable_log_stats"] = True
|
||||
removed_vision_keys = ("image_token_id", "image_feature_size",
|
||||
"image_input_shape", "image_input_type")
|
||||
if any(k in kwargs for k in removed_vision_keys):
|
||||
raise TypeError(
|
||||
"There is no need to pass vision-related arguments anymore.")
|
||||
engine_args = EngineArgs(
|
||||
model=model,
|
||||
tokenizer=tokenizer,
|
||||
|
||||
@@ -109,23 +109,12 @@ class OpenAIServingChat(OpenAIServing):
|
||||
"paligemma"):
|
||||
# These models do not use image tokens in the prompt
|
||||
return None
|
||||
if model_type.startswith("llava"):
|
||||
return self.tokenizer.decode(
|
||||
self.model_config.hf_config.image_token_index)
|
||||
|
||||
# The default behaviour assumes that the image token is
|
||||
# available to the tokenizer.
|
||||
# (Suitable for LLaVA, Idefics2, DeepSeek-VL)
|
||||
vlm_config = self.model_config.multimodal_config
|
||||
if vlm_config is None:
|
||||
raise ValueError(
|
||||
"'image_url' input is not supported as the loaded "
|
||||
"model is not multimodal.")
|
||||
|
||||
image_token_id = vlm_config.image_token_id
|
||||
if vlm_config.image_token_id is None:
|
||||
raise ValueError(
|
||||
"'image_url' input is not supported as the loaded "
|
||||
"model does not specify an image token.")
|
||||
|
||||
return self.tokenizer.decode(image_token_id)
|
||||
else:
|
||||
raise TypeError("Unknown model type: {model_type}")
|
||||
|
||||
# TODO: Let user specify how to insert image tokens into prompt
|
||||
# (similar to chat template)
|
||||
|
||||
Reference in New Issue
Block a user