diff --git a/.buildkite/intel_jobs/models_multimodal_intel.yaml b/.buildkite/intel_jobs/models_multimodal_intel.yaml
index 42d429f007..8dae59f4ea 100644
--- a/.buildkite/intel_jobs/models_multimodal_intel.yaml
+++ b/.buildkite/intel_jobs/models_multimodal_intel.yaml
@@ -22,7 +22,7 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
- 'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
+ 'pip install av &&
cd tests &&
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
@@ -47,8 +47,7 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
- 'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
- cd tests &&
+ 'cd tests &&
pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
@@ -71,8 +70,7 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
- 'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
- cd tests &&
+ 'cd tests &&
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
@@ -96,7 +94,7 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
- 'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
+ 'pip install av &&
cd tests &&
pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing'
@@ -121,7 +119,7 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
- 'pip install av matplotlib ftfy git+https://github.com/TIGER-AI-Lab/Mantis.git &&
+ 'pip install av matplotlib ftfy &&
pip install open-clip-torch --no-deps &&
cd tests &&
pytest -v -s models/multimodal/processing/test_tensor_schema.py
diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml
index 327f00ba34..36c3673990 100644
--- a/.buildkite/test-amd.yaml
+++ b/.buildkite/test-amd.yaml
@@ -239,7 +239,6 @@ steps:
- vllm/
- tests/models/multimodal
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
- pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
@@ -1514,7 +1513,6 @@ steps:
- tests/models/multimodal/generation
- tests/models/multimodal/test_mapping.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
- pytest -v -s models/multimodal/test_mapping.py
@@ -1528,7 +1526,6 @@ steps:
- vllm/
- tests/models/multimodal/generation
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
@@ -1542,7 +1539,6 @@ steps:
- vllm/
- tests/models/multimodal/generation
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
- label: "Multi-Modal Models (Standard) 1: qwen2" # TBD
@@ -1556,7 +1552,6 @@ steps:
- vllm/
- tests/models/multimodal
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
@@ -1572,7 +1567,6 @@ steps:
- tests/models/multimodal/generation
- tests/models/multimodal/test_mapping.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
- pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
@@ -1587,7 +1581,6 @@ steps:
- tests/models/multimodal/generation
- tests/models/multimodal/test_mapping.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
- pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
@@ -1603,7 +1596,6 @@ steps:
- tests/models/multimodal
- tests/models/registry.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/processing/test_tensor_schema.py
- label: Multi-Modal Processor (CPU) %N # TBD
@@ -1619,7 +1611,6 @@ steps:
- tests/models/multimodal
- tests/models/registry.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
#----------------------------------------------------- mi300 · models / quantized -----------------------------------------------------#
@@ -2467,7 +2458,6 @@ steps:
- vllm/
- tests/models/multimodal
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
- pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
@@ -2978,7 +2968,6 @@ steps:
- tests/models/multimodal/generation
- tests/models/multimodal/test_mapping.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
- pytest -v -s models/multimodal/test_mapping.py
@@ -2992,7 +2981,6 @@ steps:
- vllm/
- tests/models/multimodal/generation
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
- label: Multi-Modal Models (Extended Pooling) # TBD
@@ -3018,7 +3006,6 @@ steps:
- vllm/
- tests/models/multimodal
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
@@ -3033,7 +3020,6 @@ steps:
- vllm/
- tests/models/multimodal/generation
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
- pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml
index f9879bf8bc..9ecfe01d40 100644
--- a/.buildkite/test_areas/models_multimodal.yaml
+++ b/.buildkite/test_areas/models_multimodal.yaml
@@ -10,7 +10,6 @@ steps:
- vllm/
- tests/models/multimodal
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
mirror:
@@ -27,7 +26,6 @@ steps:
- vllm/
- tests/models/multimodal
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
- pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
mirror:
@@ -44,7 +42,6 @@ steps:
- vllm/
- tests/models/multimodal
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
- pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
mirror:
@@ -61,7 +58,6 @@ steps:
- vllm/
- tests/models/multimodal
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
- pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
- pytest models/multimodal/generation/test_memory_leak.py -m core_model
@@ -83,7 +79,6 @@ steps:
- tests/models/registry.py
device: cpu-medium
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
- label: Multi-Modal Processor # 44min
@@ -95,7 +90,6 @@ steps:
- tests/models/multimodal
- tests/models/registry.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/processing/test_tensor_schema.py
- label: Multi-Modal Accuracy Eval (Small Models) # 50min
@@ -129,7 +123,6 @@ steps:
- tests/models/multimodal/generation
- tests/models/multimodal/test_mapping.py
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
- pytest -v -s models/multimodal/test_mapping.py
mirror:
@@ -146,7 +139,6 @@ steps:
- vllm/
- tests/models/multimodal/generation
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
- label: Multi-Modal Models (Extended Generation 3)
@@ -157,7 +149,6 @@ steps:
- vllm/
- tests/models/multimodal/generation
commands:
- - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
- label: Multi-Modal Models (Extended Pooling)
diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index 87edb27f1f..099ddb2e52 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -579,7 +579,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `Lfm2VlForConditionalGeneration` | LFM2-VL | T + I+ | `LiquidAI/LFM2-VL-450M`, `LiquidAI/LFM2-VL-3B`, `LiquidAI/LFM2-VL-8B-A1B`, etc. | ✅︎ | ✅︎ |
| `Llama4ForConditionalGeneration` | Llama 4 | T + I+ | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ |
| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + IE+ | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
-| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + IE+ | `llava-hf/llava-1.5-7b-hf`, `TIGER-Lab/Mantis-8B-siglip-llama3` (see note), `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
+| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + IE+ | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
| `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + IE+ | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ |
| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ |
| `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I+ + V+ | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ |
@@ -678,9 +678,6 @@ Some models are supported only via the [Transformers modeling backend](#transfor
coordinate decoding and are not exposed by this vLLM implementation.
See [Moondream3 prompt recipes](../features/multimodal_inputs.md#moondream3-prompt-recipes).
-!!! note
- To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
-
!!! note
The official `openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (`HwwwH/MiniCPM-V-2`) for now.
For more details, please see:
diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py
index 30d34ccc61..46765ca038 100644
--- a/examples/generate/multimodal/vision_language_offline.py
+++ b/examples/generate/multimodal/vision_language_offline.py
@@ -1377,28 +1377,6 @@ def run_llava_onevision(questions: list[str], modality: str) -> ModelRequestData
)
-# Mantis
-def run_mantis(questions: list[str], modality: str) -> ModelRequestData:
- assert modality == "image"
-
- llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" # noqa: E501
- prompts = [llama3_template.format(f"{question}\n") for question in questions]
-
- engine_args = EngineArgs(
- model="TIGER-Lab/Mantis-8B-siglip-llama3",
- max_model_len=4096,
- hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
- limit_mm_per_prompt={modality: 1},
- )
- stop_token_ids = [128009]
-
- return ModelRequestData(
- engine_args=engine_args,
- prompts=prompts,
- stop_token_ids=stop_token_ids,
- )
-
-
# MiniCPM-V
def run_minicpmv_base(questions: list[str], modality: str, model_name):
assert modality in ["image", "video", "image+video"]
@@ -2390,7 +2368,6 @@ model_example_map = {
"llava-next": run_llava_next,
"llava-next-video": run_llava_next_video,
"llava-onevision": run_llava_onevision,
- "mantis": run_mantis,
"minicpmo": run_minicpmo,
"minicpmv": run_minicpmv,
"mistral3": run_mistral3,
diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py
index 30e12e899e..e245d2b899 100644
--- a/tests/models/multimodal/generation/test_common.py
+++ b/tests/models/multimodal/generation/test_common.py
@@ -763,16 +763,6 @@ VLM_TEST_SETTINGS = {
auto_cls=AutoModelForImageTextToText,
vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
),
- "mantis": VLMTestInfo(
- models=["TIGER-Lab/Mantis-8B-siglip-llama3"],
- test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
- prompt_formatter=lambda img_prompt: f"<|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
- max_model_len=4096,
- get_stop_token_ids=lambda tok: [128009],
- auto_cls=AutoModelForImageTextToText,
- vllm_output_post_proc=model_utils.mantis_vllm_to_hf_output,
- patch_hf_runner=model_utils.mantis_patch_hf_runner,
- ),
"minicpmv_25": VLMTestInfo(
models=["openbmb/MiniCPM-Llama3-V-2_5"],
test_type=VLMTestType.IMAGE,
diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py
index de9114b2a3..9076935e26 100644
--- a/tests/models/multimodal/generation/vlm_utils/model_utils.py
+++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py
@@ -167,15 +167,6 @@ def llava_onevision_vllm_to_hf_output(
return hf_output_ids, hf_output_str, out_logprobs
-def mantis_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput:
- """Sanitize vllm output [mantis] to compare with hf output."""
- output_ids, output_str, out_logprobs = vllm_output
-
- hf_output_str = output_str + "<|eot_id|>"
-
- return output_ids, hf_output_str, out_logprobs
-
-
def phi3v_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput:
"""Sanitize vllm output [phi3v] to be comparable with hf output."""
_, output_str, out_logprobs = vllm_output
@@ -940,29 +931,6 @@ def _internvl_generate(
return outputs
-def mantis_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
- from mantis.models.mllava import MLlavaProcessor
-
- hf_model.processor = MLlavaProcessor.from_pretrained(hf_model.model_name)
-
- orig_generate = hf_model.model.generate
- tokenizer = hf_model.processor.tokenizer
-
- def _generate(self, *args, **kwargs):
- return orig_generate(
- *args,
- **kwargs,
- eos_token_id=[
- tokenizer.eos_token_id,
- tokenizer.convert_tokens_to_ids("<|eot_id|>"),
- ],
- )
-
- hf_model.model.generate = types.MethodType(_generate, hf_model.model)
-
- return hf_model
-
-
def minicpmv_25_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
orig_generate = hf_model.model.generate
diff --git a/tests/models/registry.py b/tests/models/registry.py
index 14bc15ee14..34f8ca4f82 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -1059,12 +1059,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"LlavaOnevisionForConditionalGeneration": _HfExamplesInfo(
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
),
- "MantisForConditionalGeneration": _HfExamplesInfo(
- "TIGER-Lab/Mantis-8B-siglip-llama3",
- max_transformers_version="4.48",
- transformers_version_reason={"hf": "HF model is not compatible."},
- hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
- ),
"MiDashengLMModel": _HfExamplesInfo(
"mispeech/midashenglm-7b", trust_remote_code=True
),
diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py
index 2a50678670..1e850a7efc 100644
--- a/vllm/model_executor/models/llava.py
+++ b/vllm/model_executor/models/llava.py
@@ -20,7 +20,7 @@ from transformers.models.pixtral import PixtralProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
-from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input
+from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -41,11 +41,9 @@ from vllm.multimodal.processing import (
BaseMultiModalProcessor,
BaseProcessingInfo,
InputProcessingContext,
- ProcessorInputs,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
- TimingContext,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -754,97 +752,3 @@ class LlavaForConditionalGeneration(
# LLaVA's MLP projector outputs the same number of tokens
# as it receives from the vision encoder (1:1 mapping)
return num_vision_tokens
-
-
-class MantisProcessingInfo(LlavaProcessingInfo):
- def get_hf_processor(self, **kwargs: object):
- hf_config = self.get_hf_config()
- vision_info = self.get_vision_encoder_info()
-
- kwargs.setdefault("patch_size", vision_info.get_patch_size())
- kwargs.setdefault(
- "vision_feature_select_strategy",
- hf_config.vision_feature_select_strategy,
- )
-
- return self.ctx.get_hf_processor(LlavaProcessor, **kwargs)
-
-
-class MantisMultiModalProcessor(LlavaMultiModalProcessor):
- def apply(
- self,
- inputs: ProcessorInputs,
- timing_ctx: TimingContext,
- ) -> MultiModalInput:
- hf_config = self.info.get_hf_config()
- image_token_id = hf_config.image_token_index
-
- # Assume that it doesn't depend on the image size
- num_image_tokens = self.info.get_num_image_tokens(
- image_width=-1,
- image_height=-1,
- )
-
- result = super().apply(inputs, timing_ctx)
-
- mm_item_counts = inputs.mm_data_items.get_all_counts()
- mm_kwargs = result["mm_kwargs"]
- mm_hashes = result["mm_hashes"]
-
- # We reimplement the functionality of MLlavaProcessor from
- # https://github.com/TIGER-AI-Lab/Mantis.git
- def get_replacement_mantis(item_idx: int):
- return "".join(
- [
- f"(image {item_idx + 1}: ", # 7 tokens
- "" * num_image_tokens,
- ")", # 3 tokens
- ]
- )
-
- mantis_mm_repls = self._bind_and_group_updates(
- [
- PromptReplacement(
- modality="image",
- target=[image_token_id] * num_image_tokens,
- replacement=get_replacement_mantis,
- )
- ],
- mm_item_counts,
- )
-
- prompt_ids, _ = self._apply_prompt_updates(
- result["prompt_token_ids"],
- mantis_mm_repls,
- )
-
- orig_repls = self._get_mm_prompt_updates(
- inputs.mm_data_items,
- inputs.hf_processor_mm_kwargs,
- mm_kwargs,
- )
- mm_placeholders = self._find_mm_placeholders(prompt_ids, orig_repls)
- self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
-
- mm_placeholder_ranges = {
- modality: [item.to_range() for item in placeholders]
- for modality, placeholders in mm_placeholders.items()
- }
-
- return mm_input(
- prompt_token_ids=prompt_ids,
- mm_kwargs=mm_kwargs,
- mm_hashes=mm_hashes,
- mm_placeholders=mm_placeholder_ranges,
- )
-
-
-# To use this model, please use
-# `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
-@MULTIMODAL_REGISTRY.register_processor(
- MantisMultiModalProcessor,
- info=MantisProcessingInfo,
- dummy_inputs=LlavaDummyInputsBuilder,
-)
-class MantisForConditionalGeneration(LlavaForConditionalGeneration):
- pass
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index 283cd219e9..5603a0bb1b 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -477,7 +477,6 @@ _MULTIMODAL_MODELS = {
"llava_onevision",
"LlavaOnevisionForConditionalGeneration",
),
- "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"),
"MiDashengLMModel": ("midashenglm", "MiDashengLMModel"),
"MiMoV2OmniForCausalLM": ("mimo_v2_omni", "MiMoV2OmniForCausalLM"),
"MiniMaxM3SparseForConditionalGeneration": (
@@ -733,6 +732,7 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
"Grok1ForCausalLM": "0.24.0",
"TarsierForConditionalGeneration": "0.24.0",
"Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4
+ "MantisForConditionalGeneration": "0.24.0",
}
_OOT_SUPPORTED_MODELS = {