[Core][VLM] Test registration for OOT multimodal models (#8717)
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
DarkLight1337
parent
e5dc713c23
commit
26aa325f4f
+25
-5
@@ -879,15 +879,16 @@ def num_gpus_available():
|
||||
|
||||
|
||||
temp_dir = tempfile.gettempdir()
|
||||
_dummy_path = os.path.join(temp_dir, "dummy_opt")
|
||||
_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
|
||||
_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_opt_path():
|
||||
json_path = os.path.join(_dummy_path, "config.json")
|
||||
if not os.path.exists(_dummy_path):
|
||||
json_path = os.path.join(_dummy_opt_path, "config.json")
|
||||
if not os.path.exists(_dummy_opt_path):
|
||||
snapshot_download(repo_id="facebook/opt-125m",
|
||||
local_dir=_dummy_path,
|
||||
local_dir=_dummy_opt_path,
|
||||
ignore_patterns=[
|
||||
"*.bin", "*.bin.index.json", "*.pt", "*.h5",
|
||||
"*.msgpack"
|
||||
@@ -898,4 +899,23 @@ def dummy_opt_path():
|
||||
config["architectures"] = ["MyOPTForCausalLM"]
|
||||
with open(json_path, "w") as f:
|
||||
json.dump(config, f)
|
||||
return _dummy_path
|
||||
return _dummy_opt_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_llava_path():
|
||||
json_path = os.path.join(_dummy_llava_path, "config.json")
|
||||
if not os.path.exists(_dummy_llava_path):
|
||||
snapshot_download(repo_id="llava-hf/llava-1.5-7b-hf",
|
||||
local_dir=_dummy_llava_path,
|
||||
ignore_patterns=[
|
||||
"*.bin", "*.bin.index.json", "*.pt", "*.h5",
|
||||
"*.msgpack"
|
||||
])
|
||||
assert os.path.exists(json_path)
|
||||
with open(json_path, "r") as f:
|
||||
config = json.load(f)
|
||||
config["architectures"] = ["MyLlava"]
|
||||
with open(json_path, "w") as f:
|
||||
json.dump(config, f)
|
||||
return _dummy_llava_path
|
||||
|
||||
@@ -21,7 +21,9 @@ def server():
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"5",
|
||||
"--enforce-eager",
|
||||
]
|
||||
|
||||
|
||||
@@ -23,9 +23,16 @@ TEST_IMAGE_URLS = [
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--dtype", "bfloat16", "--max-model-len", "4096", "--max-num-seqs",
|
||||
"5", "--enforce-eager", "--trust-remote-code", "--limit-mm-per-prompt",
|
||||
f"image={MAXIMUM_IMAGES}"
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--max-num-seqs",
|
||||
"5",
|
||||
"--enforce-eager",
|
||||
"--trust-remote-code",
|
||||
"--limit-mm-per-prompt",
|
||||
f"image={MAXIMUM_IMAGES}",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
|
||||
@@ -3,6 +3,7 @@ import os
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
from ..utils import fork_new_process_for_each_test
|
||||
|
||||
@@ -29,3 +30,40 @@ def test_oot_registration(dummy_opt_path):
|
||||
# make sure only the first token is generated
|
||||
rest = generated_text.replace(first_token, "")
|
||||
assert rest == ""
|
||||
|
||||
|
||||
image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
|
||||
|
||||
|
||||
@fork_new_process_for_each_test
|
||||
def test_oot_multimodal_registration(dummy_llava_path):
|
||||
os.environ["VLLM_PLUGINS"] = "register_dummy_model"
|
||||
prompts = [{
|
||||
"prompt": "What's in the image?<image>",
|
||||
"multi_modal_data": {
|
||||
"image": image
|
||||
},
|
||||
}, {
|
||||
"prompt": "Describe the image<image>",
|
||||
"multi_modal_data": {
|
||||
"image": image
|
||||
},
|
||||
}]
|
||||
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
llm = LLM(model=dummy_llava_path,
|
||||
load_format="dummy",
|
||||
max_num_seqs=1,
|
||||
trust_remote_code=True,
|
||||
gpu_memory_utilization=0.98,
|
||||
max_model_len=4096,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"image": 1})
|
||||
first_token = llm.get_tokenizer().decode(0)
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
for output in outputs:
|
||||
generated_text = output.outputs[0].text
|
||||
# make sure only the first token is generated
|
||||
rest = generated_text.replace(first_token, "")
|
||||
assert rest == ""
|
||||
|
||||
@@ -1,26 +1,14 @@
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import ModelRegistry
|
||||
from vllm.model_executor.models.opt import OPTForCausalLM
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
|
||||
|
||||
class MyOPTForCausalLM(OPTForCausalLM):
|
||||
|
||||
def compute_logits(
|
||||
self, hidden_states: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
|
||||
# this dummy model always predicts the first token
|
||||
logits = super().compute_logits(hidden_states, sampling_metadata)
|
||||
if logits is not None:
|
||||
logits.zero_()
|
||||
logits[:, 0] += 1.0
|
||||
return logits
|
||||
|
||||
|
||||
def register():
|
||||
# register our dummy model
|
||||
# Test directly passing the model
|
||||
from .my_opt import MyOPTForCausalLM
|
||||
|
||||
if "MyOPTForCausalLM" not in ModelRegistry.get_supported_archs():
|
||||
ModelRegistry.register_model("MyOPTForCausalLM", MyOPTForCausalLM)
|
||||
|
||||
# Test passing lazy model
|
||||
if "MyLlava" not in ModelRegistry.get_supported_archs():
|
||||
ModelRegistry.register_model("MyLlava",
|
||||
"vllm_add_dummy_model.my_llava:MyLlava")
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.inputs import INPUT_REGISTRY
|
||||
from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
|
||||
dummy_data_for_llava,
|
||||
get_max_llava_image_tokens,
|
||||
input_processor_for_llava)
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_image_input_mapper()
|
||||
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_image_tokens)
|
||||
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
|
||||
@INPUT_REGISTRY.register_input_processor(input_processor_for_llava)
|
||||
class MyLlava(LlavaForConditionalGeneration):
|
||||
|
||||
def compute_logits(
|
||||
self, hidden_states: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
|
||||
# this dummy model always predicts the first token
|
||||
logits = super().compute_logits(hidden_states, sampling_metadata)
|
||||
if logits is not None:
|
||||
logits.zero_()
|
||||
logits[:, 0] += 1.0
|
||||
return logits
|
||||
@@ -0,0 +1,19 @@
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.opt import OPTForCausalLM
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
|
||||
|
||||
class MyOPTForCausalLM(OPTForCausalLM):
|
||||
|
||||
def compute_logits(
|
||||
self, hidden_states: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata) -> Optional[torch.Tensor]:
|
||||
# this dummy model always predicts the first token
|
||||
logits = super().compute_logits(hidden_states, sampling_metadata)
|
||||
if logits is not None:
|
||||
logits.zero_()
|
||||
logits[:, 0] += 1.0
|
||||
return logits
|
||||
Reference in New Issue
Block a user