From 4470ae84de525640a6ed8701a28882ff392e06b6 Mon Sep 17 00:00:00 2001 From: Tiezhen WANG <38108242+xianbaoqian@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:13:58 +0800 Subject: [PATCH] Remove mantis (#46806) Signed-off-by: Xianbao QIAN Co-authored-by: Claude --- .../intel_jobs/models_multimodal_intel.yaml | 12 +-- .buildkite/test-amd.yaml | 14 --- .buildkite/test_areas/models_multimodal.yaml | 9 -- docs/models/supported_models.md | 5 +- .../multimodal/vision_language_offline.py | 23 ----- .../multimodal/generation/test_common.py | 10 -- .../generation/vlm_utils/model_utils.py | 32 ------ tests/models/registry.py | 6 -- vllm/model_executor/models/llava.py | 98 +------------------ vllm/model_executor/models/registry.py | 2 +- 10 files changed, 8 insertions(+), 203 deletions(-) diff --git a/.buildkite/intel_jobs/models_multimodal_intel.yaml b/.buildkite/intel_jobs/models_multimodal_intel.yaml index 42d429f007..8dae59f4ea 100644 --- a/.buildkite/intel_jobs/models_multimodal_intel.yaml +++ b/.buildkite/intel_jobs/models_multimodal_intel.yaml @@ -22,7 +22,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git && + 'pip install av && cd tests && pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" && pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model' @@ -47,8 +47,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git && - cd tests && + 'cd tests && pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model' - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" @@ -71,8 +70,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git && - cd tests && + 'cd tests && pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" && pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model' @@ -96,7 +94,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git && + 'pip install av && cd tests && pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing' @@ -121,7 +119,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install av matplotlib ftfy git+https://github.com/TIGER-AI-Lab/Mantis.git && + 'pip install av matplotlib ftfy && pip install open-clip-torch --no-deps && cd tests && pytest -v -s models/multimodal/processing/test_tensor_schema.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 327f00ba34..36c3673990 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -239,7 +239,6 @@ steps: - vllm/ - tests/models/multimodal commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model @@ -1514,7 +1513,6 @@ steps: - tests/models/multimodal/generation - tests/models/multimodal/test_mapping.py commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py - pytest -v -s models/multimodal/test_mapping.py @@ -1528,7 +1526,6 @@ steps: - vllm/ - tests/models/multimodal/generation commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' @@ -1542,7 +1539,6 @@ steps: - vllm/ - tests/models/multimodal/generation commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' - label: "Multi-Modal Models (Standard) 1: qwen2" # TBD @@ -1556,7 +1552,6 @@ steps: - vllm/ - tests/models/multimodal commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model @@ -1572,7 +1567,6 @@ steps: - tests/models/multimodal/generation - tests/models/multimodal/test_mapping.py commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model @@ -1587,7 +1581,6 @@ steps: - tests/models/multimodal/generation - tests/models/multimodal/test_mapping.py commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model @@ -1603,7 +1596,6 @@ steps: - tests/models/multimodal - tests/models/registry.py commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/processing/test_tensor_schema.py - label: Multi-Modal Processor (CPU) %N # TBD @@ -1619,7 +1611,6 @@ steps: - tests/models/multimodal - tests/models/registry.py commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB #----------------------------------------------------- mi300 · models / quantized -----------------------------------------------------# @@ -2467,7 +2458,6 @@ steps: - vllm/ - tests/models/multimodal commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma" - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model @@ -2978,7 +2968,6 @@ steps: - tests/models/multimodal/generation - tests/models/multimodal/test_mapping.py commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py - pytest -v -s models/multimodal/test_mapping.py @@ -2992,7 +2981,6 @@ steps: - vllm/ - tests/models/multimodal/generation commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' - label: Multi-Modal Models (Extended Pooling) # TBD @@ -3018,7 +3006,6 @@ steps: - vllm/ - tests/models/multimodal commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model @@ -3033,7 +3020,6 @@ steps: - vllm/ - tests/models/multimodal/generation commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index f9879bf8bc..9ecfe01d40 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -10,7 +10,6 @@ steps: - vllm/ - tests/models/multimodal commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model mirror: @@ -27,7 +26,6 @@ steps: - vllm/ - tests/models/multimodal commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma" - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model mirror: @@ -44,7 +42,6 @@ steps: - vllm/ - tests/models/multimodal commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model mirror: @@ -61,7 +58,6 @@ steps: - vllm/ - tests/models/multimodal commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model - pytest models/multimodal/generation/test_memory_leak.py -m core_model @@ -83,7 +79,6 @@ steps: - tests/models/registry.py device: cpu-medium commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py - label: Multi-Modal Processor # 44min @@ -95,7 +90,6 @@ steps: - tests/models/multimodal - tests/models/registry.py commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/processing/test_tensor_schema.py - label: Multi-Modal Accuracy Eval (Small Models) # 50min @@ -129,7 +123,6 @@ steps: - tests/models/multimodal/generation - tests/models/multimodal/test_mapping.py commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py - pytest -v -s models/multimodal/test_mapping.py mirror: @@ -146,7 +139,6 @@ steps: - vllm/ - tests/models/multimodal/generation commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model' - label: Multi-Modal Models (Extended Generation 3) @@ -157,7 +149,6 @@ steps: - vllm/ - tests/models/multimodal/generation commands: - - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model' - label: Multi-Modal Models (Extended Pooling) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 87edb27f1f..099ddb2e52 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -579,7 +579,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `Lfm2VlForConditionalGeneration` | LFM2-VL | T + I+ | `LiquidAI/LFM2-VL-450M`, `LiquidAI/LFM2-VL-3B`, `LiquidAI/LFM2-VL-8B-A1B`, etc. | ✅︎ | ✅︎ | | `Llama4ForConditionalGeneration` | Llama 4 | T + I+ | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ | | `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + IE+ | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ | -| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + IE+ | `llava-hf/llava-1.5-7b-hf`, `TIGER-Lab/Mantis-8B-siglip-llama3` (see note), `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ | +| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + IE+ | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ | | `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + IE+ | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ | | `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ | | `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I+ + V+ | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ | @@ -678,9 +678,6 @@ Some models are supported only via the [Transformers modeling backend](#transfor coordinate decoding and are not exposed by this vLLM implementation. See [Moondream3 prompt recipes](../features/multimodal_inputs.md#moondream3-prompt-recipes). -!!! note - To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM. - !!! note The official `openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (`HwwwH/MiniCPM-V-2`) for now. For more details, please see: diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py index 30d34ccc61..46765ca038 100644 --- a/examples/generate/multimodal/vision_language_offline.py +++ b/examples/generate/multimodal/vision_language_offline.py @@ -1377,28 +1377,6 @@ def run_llava_onevision(questions: list[str], modality: str) -> ModelRequestData ) -# Mantis -def run_mantis(questions: list[str], modality: str) -> ModelRequestData: - assert modality == "image" - - llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" # noqa: E501 - prompts = [llama3_template.format(f"{question}\n") for question in questions] - - engine_args = EngineArgs( - model="TIGER-Lab/Mantis-8B-siglip-llama3", - max_model_len=4096, - hf_overrides={"architectures": ["MantisForConditionalGeneration"]}, - limit_mm_per_prompt={modality: 1}, - ) - stop_token_ids = [128009] - - return ModelRequestData( - engine_args=engine_args, - prompts=prompts, - stop_token_ids=stop_token_ids, - ) - - # MiniCPM-V def run_minicpmv_base(questions: list[str], modality: str, model_name): assert modality in ["image", "video", "image+video"] @@ -2390,7 +2368,6 @@ model_example_map = { "llava-next": run_llava_next, "llava-next-video": run_llava_next_video, "llava-onevision": run_llava_onevision, - "mantis": run_mantis, "minicpmo": run_minicpmo, "minicpmv": run_minicpmv, "mistral3": run_mistral3, diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index 30e12e899e..e245d2b899 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -763,16 +763,6 @@ VLM_TEST_SETTINGS = { auto_cls=AutoModelForImageTextToText, vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output, ), - "mantis": VLMTestInfo( - models=["TIGER-Lab/Mantis-8B-siglip-llama3"], - test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), - prompt_formatter=lambda img_prompt: f"<|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501 - max_model_len=4096, - get_stop_token_ids=lambda tok: [128009], - auto_cls=AutoModelForImageTextToText, - vllm_output_post_proc=model_utils.mantis_vllm_to_hf_output, - patch_hf_runner=model_utils.mantis_patch_hf_runner, - ), "minicpmv_25": VLMTestInfo( models=["openbmb/MiniCPM-Llama3-V-2_5"], test_type=VLMTestType.IMAGE, diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index de9114b2a3..9076935e26 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -167,15 +167,6 @@ def llava_onevision_vllm_to_hf_output( return hf_output_ids, hf_output_str, out_logprobs -def mantis_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput: - """Sanitize vllm output [mantis] to compare with hf output.""" - output_ids, output_str, out_logprobs = vllm_output - - hf_output_str = output_str + "<|eot_id|>" - - return output_ids, hf_output_str, out_logprobs - - def phi3v_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput: """Sanitize vllm output [phi3v] to be comparable with hf output.""" _, output_str, out_logprobs = vllm_output @@ -940,29 +931,6 @@ def _internvl_generate( return outputs -def mantis_patch_hf_runner(hf_model: HfRunner) -> HfRunner: - from mantis.models.mllava import MLlavaProcessor - - hf_model.processor = MLlavaProcessor.from_pretrained(hf_model.model_name) - - orig_generate = hf_model.model.generate - tokenizer = hf_model.processor.tokenizer - - def _generate(self, *args, **kwargs): - return orig_generate( - *args, - **kwargs, - eos_token_id=[ - tokenizer.eos_token_id, - tokenizer.convert_tokens_to_ids("<|eot_id|>"), - ], - ) - - hf_model.model.generate = types.MethodType(_generate, hf_model.model) - - return hf_model - - def minicpmv_25_patch_hf_runner(hf_model: HfRunner) -> HfRunner: orig_generate = hf_model.model.generate diff --git a/tests/models/registry.py b/tests/models/registry.py index 14bc15ee14..34f8ca4f82 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1059,12 +1059,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo( "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" ), - "MantisForConditionalGeneration": _HfExamplesInfo( - "TIGER-Lab/Mantis-8B-siglip-llama3", - max_transformers_version="4.48", - transformers_version_reason={"hf": "HF model is not compatible."}, - hf_overrides={"architectures": ["MantisForConditionalGeneration"]}, - ), "MiDashengLMModel": _HfExamplesInfo( "mispeech/midashenglm-7b", trust_remote_code=True ), diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index 2a50678670..1e850a7efc 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -20,7 +20,7 @@ from transformers.models.pixtral import PixtralProcessor from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions -from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input +from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.quantization import QuantizationConfig @@ -41,11 +41,9 @@ from vllm.multimodal.processing import ( BaseMultiModalProcessor, BaseProcessingInfo, InputProcessingContext, - ProcessorInputs, PromptReplacement, PromptUpdate, PromptUpdateDetails, - TimingContext, ) from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -754,97 +752,3 @@ class LlavaForConditionalGeneration( # LLaVA's MLP projector outputs the same number of tokens # as it receives from the vision encoder (1:1 mapping) return num_vision_tokens - - -class MantisProcessingInfo(LlavaProcessingInfo): - def get_hf_processor(self, **kwargs: object): - hf_config = self.get_hf_config() - vision_info = self.get_vision_encoder_info() - - kwargs.setdefault("patch_size", vision_info.get_patch_size()) - kwargs.setdefault( - "vision_feature_select_strategy", - hf_config.vision_feature_select_strategy, - ) - - return self.ctx.get_hf_processor(LlavaProcessor, **kwargs) - - -class MantisMultiModalProcessor(LlavaMultiModalProcessor): - def apply( - self, - inputs: ProcessorInputs, - timing_ctx: TimingContext, - ) -> MultiModalInput: - hf_config = self.info.get_hf_config() - image_token_id = hf_config.image_token_index - - # Assume that it doesn't depend on the image size - num_image_tokens = self.info.get_num_image_tokens( - image_width=-1, - image_height=-1, - ) - - result = super().apply(inputs, timing_ctx) - - mm_item_counts = inputs.mm_data_items.get_all_counts() - mm_kwargs = result["mm_kwargs"] - mm_hashes = result["mm_hashes"] - - # We reimplement the functionality of MLlavaProcessor from - # https://github.com/TIGER-AI-Lab/Mantis.git - def get_replacement_mantis(item_idx: int): - return "".join( - [ - f"(image {item_idx + 1}: ", # 7 tokens - "" * num_image_tokens, - ")", # 3 tokens - ] - ) - - mantis_mm_repls = self._bind_and_group_updates( - [ - PromptReplacement( - modality="image", - target=[image_token_id] * num_image_tokens, - replacement=get_replacement_mantis, - ) - ], - mm_item_counts, - ) - - prompt_ids, _ = self._apply_prompt_updates( - result["prompt_token_ids"], - mantis_mm_repls, - ) - - orig_repls = self._get_mm_prompt_updates( - inputs.mm_data_items, - inputs.hf_processor_mm_kwargs, - mm_kwargs, - ) - mm_placeholders = self._find_mm_placeholders(prompt_ids, orig_repls) - self._validate_mm_placeholders(mm_placeholders, mm_item_counts) - - mm_placeholder_ranges = { - modality: [item.to_range() for item in placeholders] - for modality, placeholders in mm_placeholders.items() - } - - return mm_input( - prompt_token_ids=prompt_ids, - mm_kwargs=mm_kwargs, - mm_hashes=mm_hashes, - mm_placeholders=mm_placeholder_ranges, - ) - - -# To use this model, please use -# `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` -@MULTIMODAL_REGISTRY.register_processor( - MantisMultiModalProcessor, - info=MantisProcessingInfo, - dummy_inputs=LlavaDummyInputsBuilder, -) -class MantisForConditionalGeneration(LlavaForConditionalGeneration): - pass diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 283cd219e9..5603a0bb1b 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -477,7 +477,6 @@ _MULTIMODAL_MODELS = { "llava_onevision", "LlavaOnevisionForConditionalGeneration", ), - "MantisForConditionalGeneration": ("llava", "MantisForConditionalGeneration"), "MiDashengLMModel": ("midashenglm", "MiDashengLMModel"), "MiMoV2OmniForCausalLM": ("mimo_v2_omni", "MiMoV2OmniForCausalLM"), "MiniMaxM3SparseForConditionalGeneration": ( @@ -733,6 +732,7 @@ _PREVIOUSLY_SUPPORTED_MODELS = { "Grok1ForCausalLM": "0.24.0", "TarsierForConditionalGeneration": "0.24.0", "Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4 + "MantisForConditionalGeneration": "0.24.0", } _OOT_SUPPORTED_MODELS = {