[Frontend] Require flag for loading text and image embeds (#27204)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
DarkLight1337
parent
db6f28d898
commit
58fab50d82
@@ -811,6 +811,10 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
allowed_media_domains=tracker.allowed_media_domains,
|
||||
)
|
||||
|
||||
@property
|
||||
def model_config(self) -> ModelConfig:
|
||||
return self._tracker.model_config
|
||||
|
||||
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
|
||||
image = self._connector.fetch_image(image_url) if image_url else None
|
||||
|
||||
@@ -822,6 +826,12 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
image_embeds: str | dict[str, str] | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
if not mm_config.enable_mm_embeds:
|
||||
raise ValueError(
|
||||
"You must set `--enable-mm-embeds` to input `image_embeds`"
|
||||
)
|
||||
|
||||
if isinstance(image_embeds, dict):
|
||||
embeds = {
|
||||
k: self._connector.fetch_image_embedding(v)
|
||||
@@ -886,6 +896,10 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
allowed_media_domains=tracker.allowed_media_domains,
|
||||
)
|
||||
|
||||
@property
|
||||
def model_config(self) -> ModelConfig:
|
||||
return self._tracker.model_config
|
||||
|
||||
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
|
||||
image_coro = self._connector.fetch_image_async(image_url) if image_url else None
|
||||
|
||||
@@ -897,6 +911,12 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
image_embeds: str | dict[str, str] | None,
|
||||
uuid: str | None = None,
|
||||
) -> None:
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
if not mm_config.enable_mm_embeds:
|
||||
raise ValueError(
|
||||
"You must set `--enable-mm-embeds` to input `image_embeds`"
|
||||
)
|
||||
|
||||
future: asyncio.Future[str | dict[str, str] | None] = asyncio.Future()
|
||||
|
||||
if isinstance(image_embeds, dict):
|
||||
|
||||
@@ -156,14 +156,17 @@ class BaseRenderer(ABC):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def load_prompt_embeds(
|
||||
cls,
|
||||
self,
|
||||
prompt_embeds: bytes | list[bytes],
|
||||
truncate_prompt_tokens: Annotated[int, Field(ge=0)] | None = None,
|
||||
cache_salt: str | None = None,
|
||||
) -> list[EngineEmbedsPrompt]:
|
||||
"""Load and validate base64-encoded embeddings into prompt objects."""
|
||||
if not self.model_config.enable_prompt_embeds:
|
||||
raise ValueError(
|
||||
"You must set `--enable-prompt-embeds` to input `prompt_embeds`."
|
||||
)
|
||||
|
||||
def _load_and_validate_embed(embed: bytes) -> EngineEmbedsPrompt:
|
||||
tensor = torch.load(
|
||||
|
||||
Reference in New Issue
Block a user