Implicit language-model-only mode via limit-mm-per-prompt (#22299)
Signed-off-by: Roger Wang <hey@rogerw.me> Signed-off-by: Andy Xie <andy.xning@gmail.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: Andrew Sansom <andrew@protopia.ai> Signed-off-by: Zhiyu Cheng <zhiyuc@nvidia.com> Signed-off-by: Shu Wang <shuw@nvidia.com> Signed-off-by: Po-Han Huang <pohanh@nvidia.com> Signed-off-by: Shu Wang. <shuw@nvidia.com> Signed-off-by: XIn Li <xinli@nvidia.com> Signed-off-by: Junhao Li <junhao@ubicloud.com> Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Signed-off-by: zRzRzRzRzRzRzR <2448370773@qq.com> Signed-off-by: zitian.zhao <zitian.zhao@tencentmusic.com> Signed-off-by: zitian zhao <zitian.zhao@tencentmusic.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: iAmir97 <Amir.balwel@embeddedllm.com> Signed-off-by: iAmir97 <71513472+iAmir97@users.noreply.github.com> Signed-off-by: Linkun <github@lkchen.net> Co-authored-by: Ning Xie <andy.xning@gmail.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com> Co-authored-by: Andrew Sansom <andrew@protopia.ai> Co-authored-by: Zhiyu <zhiyuc@nvidia.com> Co-authored-by: Shu Wang <shuw@nvidia.com> Co-authored-by: XIn Li <xinli@nvidia.com> Co-authored-by: Junhao Li <streaver91@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com> Co-authored-by: Yuxuan Zhang <2448370773@qq.com> Co-authored-by: ZiTian Zhao <zitian.zhao@tencentmusic.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: iAmir97 <71513472+iAmir97@users.noreply.github.com> Co-authored-by: iAmir97 <Amir.balwel@embeddedllm.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Hong Hanh <hanh.usth@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: lkchen <github@lkchen.net>
This commit is contained in:
co-authored by
Ning Xie
TJian
Andrew Sansom
Zhiyu
Shu Wang
XIn Li
Junhao Li
Chauncey
Yuxuan Zhang
ZiTian Zhao
Cyrus Leung
Po-Han Huang
iAmir97
iAmir97
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Cyrus Leung
Hong Hanh
youkaichao
lkchen
parent
429e4e2d42
commit
08b751ba74
@@ -3,7 +3,7 @@
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from vllm.multimodal import MultiModalKwargs
|
||||
from vllm.multimodal import MultiModalKwargs, MultiModalRegistry
|
||||
from vllm.multimodal.cache import MultiModalCache, MultiModalCacheItemMetadata
|
||||
from vllm.utils import is_list_of
|
||||
|
||||
@@ -46,10 +46,11 @@ if TYPE_CHECKING:
|
||||
class MultiModalInputCacheClient:
|
||||
"""Used by P0 to check whether multi-modal kwargs are cached in P1."""
|
||||
|
||||
def __init__(self, model_config: "ModelConfig") -> None:
|
||||
def __init__(self, model_config: "ModelConfig",
|
||||
mm_registry: MultiModalRegistry) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.enabled = model_config.enable_mm_input_cache
|
||||
self.enabled = mm_registry.enable_mm_input_cache(model_config)
|
||||
self.mm_cache = MultiModalCache.get_lru_cache(
|
||||
model_config.get_mm_input_cache_gb(),
|
||||
MultiModalCacheItemMetadata,
|
||||
@@ -85,10 +86,11 @@ class MultiModalInputCacheClient:
|
||||
class MultiModalInputCacheServer:
|
||||
"""Used by P1 to avoid requiring past multi-modal kwargs from P0."""
|
||||
|
||||
def __init__(self, model_config: "ModelConfig") -> None:
|
||||
def __init__(self, model_config: "ModelConfig",
|
||||
mm_registry: MultiModalRegistry) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.enabled = model_config.enable_mm_input_cache
|
||||
self.enabled = mm_registry.enable_mm_input_cache(model_config)
|
||||
self.mm_cache = MultiModalCache.get_lru_cache(
|
||||
model_config.get_mm_input_cache_gb(),
|
||||
MultiModalKwargs,
|
||||
|
||||
Reference in New Issue
Block a user