From 7198940b395e74337101e7b62437c55a7e1d4284 Mon Sep 17 00:00:00 2001 From: Dong W <89223086+sniper35@users.noreply.github.com> Date: Thu, 30 Apr 2026 19:06:48 -0700 Subject: [PATCH] [Model] Add Moondream3 model support(only query and caption skills) (#32325) Signed-off-by: Dong Wang --- docs/features/multimodal_inputs.md | 61 + docs/models/supported_models.md | 7 + tests/conftest.py | 39 +- .../multimodal/generation/test_common.py | 26 + .../multimodal/generation/test_moondream3.py | 176 ++ .../multimodal/generation/vlm_utils/core.py | 13 +- .../generation/vlm_utils/model_utils.py | 218 +++ .../multimodal/generation/vlm_utils/types.py | 2 + .../multimodal/processing/test_moondream3.py | 553 +++++++ tests/models/registry.py | 10 + tests/models/utils.py | 14 + vllm/model_executor/models/moondream3.py | 1423 +++++++++++++++++ vllm/model_executor/models/registry.py | 2 + vllm/transformers_utils/config.py | 1 + vllm/transformers_utils/configs/__init__.py | 6 + vllm/transformers_utils/configs/moondream3.py | 152 ++ .../model_arch_config_convertor.py | 1 + .../transformers_utils/processors/__init__.py | 2 + .../processors/moondream3.py | 541 +++++++ 19 files changed, 3238 insertions(+), 9 deletions(-) create mode 100644 tests/models/multimodal/generation/test_moondream3.py create mode 100644 tests/models/multimodal/processing/test_moondream3.py create mode 100644 vllm/model_executor/models/moondream3.py create mode 100644 vllm/transformers_utils/configs/moondream3.py create mode 100644 vllm/transformers_utils/processors/moondream3.py diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 33796e20e7..f6d4f3f86d 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -215,6 +215,67 @@ When loading RGBA images (images with transparency), vLLM converts them to RGB f - This setting only affects RGBA images with transparency; RGB images are unchanged - If not specified, the default white background `(255, 255, 255)` is used for backward compatibility +#### Moondream3 Prompt Recipes { #moondream3-prompt-recipes } + +`Moondream3ForCausalLM` supports two task-specific prompt formats: + +- `query`: ask a question about the image. +- `caption`: generate a caption for the image. + +```python +from vllm import LLM, SamplingParams +from vllm.assets.image import ImageAsset + +llm = LLM( + model="moondream/moondream3-preview", + tokenizer="moondream/starmie-v1", + trust_remote_code=True, + max_model_len=2048, + limit_mm_per_prompt={"image": 1}, +) + +image = ImageAsset("stop_sign").pil_image + + +def make_query_prompt(question: str) -> str: + return ( + "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>" + f"{question}<|md_reserved_2|>" + ) + + +def make_caption_prompt(length: str = "normal") -> str: + return ( + "<|endoftext|><|md_reserved_0|>" + f"describe<|md_reserved_1|>{length}<|md_reserved_2|>" + ) + + +query_out = llm.generate( + { + "prompt": make_query_prompt("What is shown in this image?"), + "multi_modal_data": {"image": image}, + }, + SamplingParams(max_tokens=64, temperature=0), +)[0].outputs[0].text + +caption_out = llm.generate( + { + "prompt": make_caption_prompt(), + "multi_modal_data": {"image": image}, + }, + SamplingParams(max_tokens=100, temperature=0), +)[0].outputs[0].text + +print("query:", query_out) +print("caption:", caption_out) +``` + +!!! note + The native Moondream3 model also has `detect` and `point` skills. Those + require custom coordinate decoding and are not exposed by this vLLM + implementation. + ### Video Inputs You can pass a list of NumPy arrays directly to the `'video'` field of the multi-modal dictionary diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 6678cc8fb0..97c08a8b62 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -599,6 +599,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I+ | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ | | `MolmoForCausalLM` | Molmo | T + I+ | `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc. | ✅︎ | ✅︎ | | `Molmo2ForConditionalGeneration` | Molmo2 | T + I+ / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B` | ✅︎ | ✅︎ | +| `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ | | `MusicFlamingoForConditionalGeneration` | MusicFlamingo | T + A | `nvidia/music-flamingo-2601-hf`, `nvidia/music-flamingo-think-2601-hf` | ✅︎ | ✅︎ | | `NVLM_D_Model` | NVLM-D 1.0 | T + I+ | `nvidia/NVLM-D-72B`, etc. | | ✅︎ | | `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + IE+ | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ | @@ -661,6 +662,12 @@ Some models are supported only via the [Transformers modeling backend](#transfor !!! note For `InternVLChatModel`, only InternVL2.5 with Qwen2.5 text backbone (`OpenGVLab/InternVL2.5-1B` etc.), InternVL3 and InternVL3.5 have video inputs support currently. +!!! note + `Moondream3ForCausalLM` uses task-specific prompt templates for `query` + and `caption`. The native `detect` and `point` skills require custom + coordinate decoding and are not exposed by this vLLM implementation. + See [Moondream3 prompt recipes](../features/multimodal_inputs.md#moondream3-prompt-recipes). + !!! note To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM. diff --git a/tests/conftest.py b/tests/conftest.py index 9ec31d83c7..40adeda2bd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -371,6 +371,8 @@ class HfRunner: is_cross_encoder: bool = False, skip_tokenizer_init: bool = False, auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM, + tokenizer_name: str | None = None, + processor: Any | None = None, # Set this to avoid hanging issue default_torch_num_threads: int | None = None, ) -> None: @@ -391,6 +393,8 @@ class HfRunner: is_cross_encoder=is_cross_encoder, skip_tokenizer_init=skip_tokenizer_init, auto_cls=auto_cls, + tokenizer_name=tokenizer_name, + processor=processor, ) def _init( @@ -405,6 +409,8 @@ class HfRunner: is_cross_encoder: bool = False, skip_tokenizer_init: bool = False, auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM, + tokenizer_name: str | None = None, + processor: Any | None = None, ) -> None: model_name = maybe_model_redirect(model_name) self.model_name = model_name @@ -484,20 +490,27 @@ class HfRunner: if not skip_tokenizer_init: self.tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast" = ( AutoTokenizer.from_pretrained( - model_name, + tokenizer_name or model_name, trust_remote_code=trust_remote_code, ) ) - # don't put this import at the top level - # it will call torch.accelerator.device_count() - from transformers import AutoProcessor + if processor is not None: + self.processor = processor + else: + # don't put this import at the top level + # it will call torch.accelerator.device_count() + from transformers import AutoProcessor - self.processor = AutoProcessor.from_pretrained( - model_name, - trust_remote_code=trust_remote_code, - ) + self.processor = AutoProcessor.from_pretrained( + model_name, + trust_remote_code=trust_remote_code, + ) if skip_tokenizer_init: + if self.processor is None: + raise ValueError( + "skip_tokenizer_init=True requires processor initialization." + ) self.tokenizer = self.processor.tokenizer def get_inputs( @@ -520,6 +533,12 @@ class HfRunner: all_inputs: list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]] = [] for i, prompt in enumerate(prompts): if isinstance(prompt, str): + if self.processor is None: + raise RuntimeError( + "HfRunner.processor is not initialized. " + "Pass processor=... to HfRunner or set " + "hf_model.processor before generation." + ) # Create a copy to avoid modifying the original dict processor_kwargs = ( tokenization_kwargs.copy() @@ -617,6 +636,10 @@ class HfRunner: use_cache=True, **kwargs, ) + if self.processor is None: + raise RuntimeError( + "HfRunner.processor is not initialized; cannot decode output." + ) output_str = self.processor.batch_decode( output_ids, skip_special_tokens=True, diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index e04d44d4be..fdd306633e 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -805,6 +805,32 @@ VLM_TEST_SETTINGS = { max_num_seqs=2, patch_hf_runner=model_utils.molmo_patch_hf_runner, ), + "moondream3": VLMTestInfo( + models=["moondream/moondream3-preview"], + test_type=VLMTestType.IMAGE, + prompt_formatter=identity, + img_idx_to_prompt=lambda idx: "<|endoftext|>", + # Common-image coverage here targets query/caption. The native + # detect/point skills are not exposed by vLLM. + single_image_prompts=IMAGE_ASSETS.prompts( + { + "stop_sign": "<|md_reserved_0|>query<|md_reserved_1|>What is this sign?<|md_reserved_2|>", # noqa: E501 + "cherry_blossom": ( + "<|md_reserved_0|>query<|md_reserved_1|>What season is shown?<|md_reserved_2|>" # noqa: E501 + ), + } + ), + max_model_len=4096, + max_num_seqs=2, + dtype="bfloat16", + hf_processor=model_utils.moondream3_processor, + patch_hf_runner=model_utils.moondream3_patch_hf_runner, + # Single size factor to avoid GPU OOM when running multiple test + # cases sequentially (9B MoE model uses ~18 GiB per instance). + image_size_factors=[(1.0,)], + # Moondream3 is 9B params with MoE, needs significant GPU memory + marks=[large_gpu_mark(min_gb=48)], + ), "ovis1_6-gemma2": VLMTestInfo( models=["AIDC-AI/Ovis1.6-Gemma2-9B"], test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), diff --git a/tests/models/multimodal/generation/test_moondream3.py b/tests/models/multimodal/generation/test_moondream3.py new file mode 100644 index 0000000000..e4aaa5a55b --- /dev/null +++ b/tests/models/multimodal/generation/test_moondream3.py @@ -0,0 +1,176 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Generation tests for Moondream3 query and caption support.""" + +import pytest + +from tests.models.registry import HF_EXAMPLE_MODELS +from vllm.platforms import current_platform + +from ....conftest import IMAGE_ASSETS, ImageTestAssets +from ....utils import large_gpu_mark, multi_gpu_test + +MOONDREAM3_MODEL_ID = "moondream/moondream3-preview" +MOONDREAM3_TOKENIZER = "moondream/starmie-v1" + +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts( + { + "stop_sign": "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>What color is the stop sign?<|md_reserved_2|>", # noqa: E501 + "cherry_blossom": "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>What color are the flowers?<|md_reserved_2|>", # noqa: E501 + } +) + + +def make_query_prompt(question: str) -> str: + """Create a direct-answer query prompt for Moondream3.""" + return ( + "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>" + f"{question}<|md_reserved_2|>" + ) + + +def make_caption_prompt(length: str = "normal") -> str: + """Create a caption prompt for Moondream3.""" + return ( + "<|endoftext|><|md_reserved_0|>" + f"describe<|md_reserved_1|>{length}<|md_reserved_2|>" + ) + + +@multi_gpu_test(num_gpus=2) +@large_gpu_mark(min_gb=80) +def test_tensor_parallel(image_assets: ImageTestAssets): + import gc + + from vllm import LLM, SamplingParams + from vllm.distributed.parallel_state import destroy_model_parallel + + destroy_model_parallel() + gc.collect() + current_platform.empty_cache() + + llm = LLM( + model=MOONDREAM3_MODEL_ID, + tokenizer=MOONDREAM3_TOKENIZER, + trust_remote_code=True, + dtype="bfloat16", + tensor_parallel_size=2, + max_model_len=1024, + enforce_eager=True, + limit_mm_per_prompt={"image": 1}, + gpu_memory_utilization=0.45, + ) + + image = image_assets[0].pil_image + prompt = make_query_prompt("What color is the stop sign?") + + try: + outputs = llm.generate( + {"prompt": prompt, "multi_modal_data": {"image": image}}, + SamplingParams(max_tokens=20, temperature=0), + ) + + assert len(outputs) > 0 + assert outputs[0].outputs[0].text is not None + finally: + del llm + gc.collect() + current_platform.empty_cache() + + +@pytest.fixture(scope="module") +def llm(): + model_info = HF_EXAMPLE_MODELS.get_hf_info("Moondream3ForCausalLM") + model_info.check_transformers_version(on_fail="skip") + + from vllm import LLM + + try: + return LLM( + model=MOONDREAM3_MODEL_ID, + tokenizer=MOONDREAM3_TOKENIZER, + trust_remote_code=True, + dtype="bfloat16", + max_model_len=2048, + enforce_eager=True, + limit_mm_per_prompt={"image": 1}, + gpu_memory_utilization=0.45, + ) + except Exception as exc: + pytest.skip(f"Failed to load {MOONDREAM3_MODEL_ID}: {exc}") + + +@large_gpu_mark(min_gb=48) +def test_model_loading(llm): + assert llm is not None + + +@large_gpu_mark(min_gb=48) +def test_query_skill(llm, image_assets: ImageTestAssets): + from vllm import SamplingParams + + image = image_assets[0].pil_image + prompt = make_query_prompt("What color is the stop sign?") + + outputs = llm.generate( + {"prompt": prompt, "multi_modal_data": {"image": image}}, + SamplingParams(max_tokens=50, temperature=0), + ) + + output_text = outputs[0].outputs[0].text + assert output_text is not None + assert len(output_text) > 0 + + +@large_gpu_mark(min_gb=48) +def test_caption_skill(llm, image_assets: ImageTestAssets): + from vllm import SamplingParams + + image = image_assets[1].pil_image + prompt = make_caption_prompt() + + outputs = llm.generate( + {"prompt": prompt, "multi_modal_data": {"image": image}}, + SamplingParams(max_tokens=100, temperature=0), + ) + + output_text = outputs[0].outputs[0].text + assert output_text is not None + assert len(output_text) > 0 + + +@large_gpu_mark(min_gb=48) +def test_batched_inference(llm, image_assets: ImageTestAssets): + from vllm import SamplingParams + + images = [asset.pil_image for asset in image_assets] + prompts = [ + {"prompt": prompt, "multi_modal_data": {"image": img}} + for img, prompt in zip(images, HF_IMAGE_PROMPTS) + ] + + outputs = llm.generate(prompts, SamplingParams(max_tokens=50, temperature=0)) + + assert len(outputs) == len(images) + for output in outputs: + assert output.outputs[0].text is not None + assert len(output.outputs[0].text) > 0 + + +@pytest.mark.parametrize("asset_name", ["stop_sign", "cherry_blossom"]) +@large_gpu_mark(min_gb=48) +def test_image_assets(llm, image_assets: ImageTestAssets, asset_name: str): + from vllm import SamplingParams + + asset_idx = 0 if asset_name == "stop_sign" else 1 + image = image_assets[asset_idx].pil_image + prompt = HF_IMAGE_PROMPTS[asset_idx] + + outputs = llm.generate( + {"prompt": prompt, "multi_modal_data": {"image": image}}, + SamplingParams(max_tokens=50, temperature=0), + ) + + output_text = outputs[0].outputs[0].text + assert output_text is not None + assert len(output_text) > 0 diff --git a/tests/models/multimodal/generation/vlm_utils/core.py b/tests/models/multimodal/generation/vlm_utils/core.py index ae95f39586..207d3a3202 100644 --- a/tests/models/multimodal/generation/vlm_utils/core.py +++ b/tests/models/multimodal/generation/vlm_utils/core.py @@ -38,6 +38,7 @@ def run_test( limit_mm_per_prompt: dict[str, int], vllm_runner_kwargs: dict[str, Any] | None, hf_model_kwargs: dict[str, Any] | None, + hf_processor: Callable[[str], Any] | None, patch_hf_runner: Callable[[HfRunner], HfRunner] | None, runner: RunnerOption = "auto", distributed_executor_backend: str | None = None, @@ -116,8 +117,18 @@ def run_test( ) vllm_outputs_per_mm.append(vllm_output) + hf_runner_kwargs: dict[str, Any] = {} + if model_info.tokenizer: + hf_runner_kwargs["tokenizer_name"] = model_info.tokenizer + if hf_processor is not None: + hf_runner_kwargs["processor"] = hf_processor(model) + hf_model = hf_runner( - model, dtype=dtype, auto_cls=auto_cls, model_kwargs=hf_model_kwargs + model, + dtype=dtype, + auto_cls=auto_cls, + model_kwargs=hf_model_kwargs, + **hf_runner_kwargs, ) # Some models need to patch things like the model processor, e.g., internvl diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 0a692387cf..66307de731 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -1336,3 +1336,221 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner": hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign, assignment] hf_model.model.generate = patched_generate # type: ignore[method-assign] return hf_model + + +def moondream3_processor(model: str): + from vllm.transformers_utils.processors.moondream3 import Moondream3Processor + + return Moondream3Processor.from_pretrained(model, trust_remote_code=True) + + +def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner: + """Patch HfRunner for Moondream3.""" + moondream_processor = hf_model.processor + + def processor(*args, text="", images=None, **kwargs): + if images is None: + return moondream_processor(text=text, **kwargs) + + images_list = [images] if isinstance(images, Image) else images + return moondream_processor(images=images_list, text=text, **kwargs) + + hf_model.processor = processor + + # Expose the LM head for logprob extraction. + hf_model.model.get_output_embeddings = lambda: hf_model.model.model.text.lm_head + + native_model = hf_model.model.model # MoondreamModel instance + + from torch.nn import functional as F + + from vllm.model_executor.models.moondream3 import reconstruct_from_crops + + # Resolve the placeholder tokens from the tokenizer instead of hard-coding. + image_placeholder_ids = moondream_processor.tokenizer.encode( + "", add_special_tokens=False + ) + + def _normalize_tiling(tilings): + """Extract (h, w) tuple from various tiling container formats.""" + tiling = tilings + if isinstance(tiling, torch.Tensor): + tiling = tuple(tiling.squeeze().tolist()) + elif isinstance(tiling, (list, tuple)): + t0 = tiling[0] + if isinstance(t0, torch.Tensor): + tiling = tuple(t0.tolist()) + elif isinstance(t0, (list, tuple)): + tiling = tuple(t0) + return tiling + + def _encode_vision(pixel_values, tilings): + """Run preprocessed crops through vision encoder + projection.""" + device = native_model.device + dtype = native_model.vision.pos_emb.dtype + config = native_model.config + + pv = pixel_values + while pv.dim() > 4: + pv = pv.squeeze(0) + pv = pv.to(device=device, dtype=dtype) + + features = native_model._vis_enc(pv) + grid_size = config.vision.crop_size // config.vision.enc_patch_size + global_feat = features[0] + + if features.shape[0] > 1 and tilings is not None: + tiling = _normalize_tiling(tilings) + local = features[1:].view(-1, grid_size, grid_size, config.vision.enc_dim) + reconstructed = reconstruct_from_crops( + local, + tiling, + config.vision.overlap_margin, + patch_size=1, + ) + else: + reconstructed = global_feat.view( + grid_size, grid_size, config.vision.enc_dim + ) + + return native_model._vis_proj(global_feat, reconstructed) + + def _find_subsequence(seq, subseq): + """Find start index of subseq in seq, or None.""" + n = len(subseq) + for i in range(len(seq) - n + 1): + if seq[i : i + n] == subseq: + return i + return None + + def _generate( + self, + input_ids=None, + pixel_values=None, + tilings=None, + attention_mask=None, + **kwargs, + ): + max_new_tokens = kwargs.get("max_new_tokens", 128) + return_dict = kwargs.get("return_dict_in_generate", False) + output_hs = kwargs.get("output_hidden_states", False) + + if pixel_values is None: + sequences = input_ids + if return_dict: + return types.SimpleNamespace( + sequences=sequences, + hidden_states=() if output_hs else None, + ) + return sequences + + # Processor may return lists; extract the single element. + if isinstance(pixel_values, (list, tuple)): + pixel_values = pixel_values[0] + if ( + isinstance(tilings, (list, tuple)) + and tilings + and not isinstance(tilings[0], int) + ): + tilings = tilings[0] + + hf_model.model._setup_caches() + native_model.use_flex_decoding = False + + device = native_model.device + config = native_model.config + + with torch.inference_mode(): + for block in native_model.text.blocks: + block.kv_cache.k_cache.zero_() + block.kv_cache.v_cache.zero_() + + img_emb = _encode_vision(pixel_values, tilings) + + bos_emb = F.embedding( + torch.tensor([[config.tokenizer.bos_id]], device=device), + native_model.text.wte, + ) + img_input = torch.cat([bos_emb, img_emb.unsqueeze(0)], dim=1) + prefix_len = img_input.size(1) + + mask = native_model.attn_mask[:, :, :prefix_len, :] + pos_ids = torch.arange(prefix_len, dtype=torch.long, device=device) + native_model._prefill(img_input, mask, pos_ids, None) + + ids = input_ids.squeeze(0).tolist() + img_start = _find_subsequence(ids, image_placeholder_ids) + + if img_start is None: + sequences = input_ids + if return_dict: + return types.SimpleNamespace( + sequences=sequences, + hidden_states=() if output_hs else None, + ) + return sequences + + prompt_tokens = ids[img_start + len(image_placeholder_ids) :] + + if not prompt_tokens: + sequences = input_ids + if return_dict: + return types.SimpleNamespace( + sequences=sequences, + hidden_states=() if output_hs else None, + ) + return sequences + + prompt_tensor = torch.tensor([prompt_tokens], device=device) + prompt_emb = F.embedding(prompt_tensor, native_model.text.wte) + prompt_len = prompt_emb.size(1) + + mask = native_model.attn_mask[:, :, prefix_len : prefix_len + prompt_len, :] + pos_ids = torch.arange( + prefix_len, + prefix_len + prompt_len, + dtype=torch.long, + device=device, + ) + hidden = native_model._prefill(prompt_emb, mask, pos_ids, None) + pos = prefix_len + prompt_len + + hidden_last = native_model.text.post_ln(hidden[:, -1:, :]) + logits = native_model.text.lm_head(hidden_last.squeeze(1)) + + generated = [] + all_hidden_states = [] + # Record the hidden state that predicted each generated token. + prev_hs = hidden_last + for _ in range(max_new_tokens): + next_token = logits.argmax(dim=-1).item() + if next_token == 0: + break + generated.append(next_token) + if output_hs: + all_hidden_states.append((prev_hs,)) + + next_emb = F.embedding( + torch.tensor([[next_token]], device=device), + native_model.text.wte, + ) + mask = native_model.attn_mask[:, :, pos : pos + 1, :] + pos_ids_step = torch.tensor([pos], dtype=torch.long, device=device) + hidden = native_model._prefill(next_emb, mask, pos_ids_step, None) + hidden_last = native_model.text.post_ln(hidden[:, -1:, :]) + prev_hs = hidden_last + logits = native_model.text.lm_head(hidden_last.squeeze(1)) + pos += 1 + + result_ids = ids + generated + sequences = torch.tensor([result_ids], device=device) + + if return_dict: + return types.SimpleNamespace( + sequences=sequences, + hidden_states=tuple(all_hidden_states) if output_hs else None, + ) + return sequences + + hf_model.model.generate = types.MethodType(_generate, hf_model.model) + return hf_model diff --git a/tests/models/multimodal/generation/vlm_utils/types.py b/tests/models/multimodal/generation/vlm_utils/types.py index ae2f754813..af48a1479b 100644 --- a/tests/models/multimodal/generation/vlm_utils/types.py +++ b/tests/models/multimodal/generation/vlm_utils/types.py @@ -133,6 +133,7 @@ class VLMTestInfo(NamedTuple): # Exposed options for HF runner hf_model_kwargs: dict[str, Any] | None = None + hf_processor: Callable[[str], Any] | None = None # Indicates we should explicitly pass the EOS from the tokenizer use_tokenizer_eos: bool = False auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM @@ -196,6 +197,7 @@ class VLMTestInfo(NamedTuple): "comparator": self.comparator, "get_stop_token_ids": self.get_stop_token_ids, "hf_model_kwargs": self.hf_model_kwargs, + "hf_processor": self.hf_processor, "stop_str": self.stop_str, "patch_hf_runner": self.patch_hf_runner, } diff --git a/tests/models/multimodal/processing/test_moondream3.py b/tests/models/multimodal/processing/test_moondream3.py new file mode 100644 index 0000000000..a6284ae45f --- /dev/null +++ b/tests/models/multimodal/processing/test_moondream3.py @@ -0,0 +1,553 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for Moondream3 multimodal processing. + +Includes: +- Processor creation and application tests +- Image tokenization and placeholder expansion tests +- Tiling and cropping logic tests (CPU-based) +- Pixel normalization tests +""" + +import numpy as np +import pytest +import torch + +from vllm.multimodal import MULTIMODAL_REGISTRY + +from ....conftest import ImageTestAssets +from ...utils import build_model_context + +MOONDREAM3_MODEL_ID = "moondream/moondream3-preview" +# Expected multimodal prefix: BOS + 729 image tokens. +EXPECTED_IMAGE_TOKENS = 730 +# Vision encoder constants +CROP_SIZE = 378 +PATCH_SIZE = 14 +MAX_CROPS = 12 + + +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_processor_creation(model_id: str): + """Test that Moondream3 processor can be created.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + assert processor is not None + + +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_processor_apply( + image_assets: ImageTestAssets, + model_id: str, +): + """Test that Moondream3 processor can process inputs. + + NOTE: The prompt includes the leading BOS token because Moondream3 + pre-fills BOS and image embeddings together. + """ + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + prompt = "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501 + mm_data = {"image": [image_assets[0].pil_image]} + + processed_inputs = processor( + prompt, + mm_items=processor.info.parse_mm_data(mm_data), + hf_processor_mm_kwargs={}, + ) + + assert "prompt_token_ids" in processed_inputs + image_placeholders = processed_inputs["mm_placeholders"]["image"] + assert len(image_placeholders) == 1 + assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS + + +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_processor_pixel_values( + image_assets: ImageTestAssets, + model_id: str, +): + """Test that pixel values are correctly produced.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + prompt = "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501 + mm_data = {"image": [image_assets[0].pil_image]} + + processed_inputs = processor( + prompt, + mm_items=processor.info.parse_mm_data(mm_data), + hf_processor_mm_kwargs={}, + ) + + # Check mm_kwargs contains pixel_values + mm_kwargs = processed_inputs.get("mm_kwargs") + assert mm_kwargs is not None + mm_data_result = mm_kwargs.get_data() + assert "pixel_values" in mm_data_result + + # Verify pixel_values shape + pixel_values = mm_data_result["pixel_values"] + assert pixel_values.dim() == 5 # [batch, num_crops, C, H, W] + assert pixel_values.shape[2] == 3 # RGB channels + assert pixel_values.shape[3] == 378 # crop height + assert pixel_values.shape[4] == 378 # crop width + + +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_processor_image_token_expansion( + image_assets: ImageTestAssets, + model_id: str, +): + """Test that placeholder is expanded to correct number of tokens.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + prompt = "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>Describe.<|md_reserved_2|>" # noqa: E501 + mm_data = {"image": [image_assets[0].pil_image]} + + processed_inputs = processor( + prompt, + mm_items=processor.info.parse_mm_data(mm_data), + hf_processor_mm_kwargs={}, + ) + image_placeholders = processed_inputs["mm_placeholders"]["image"] + assert len(image_placeholders) == 1 + assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS + + +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_multi_crop_tiling( + model_id: str, +): + """Test that large images produce correct multi-crop tiling.""" + from PIL import Image + + from vllm.transformers_utils.processors.moondream3 import Moondream3Processor + + processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True) + + # Create a large image that requires multiple crops + large_image = Image.new("RGB", (1000, 1000), color="blue") + pixel_values, tiling = processor.preprocess_image(large_image) + + # Large images should produce more than 1x1 tiling + assert tiling[0] >= 1 and tiling[1] >= 1 + # Check that we have global crop + local crops + expected_crops = tiling[0] * tiling[1] + 1 + assert pixel_values.shape[0] == expected_crops + + +@pytest.mark.parametrize( + "image_size", + [ + (500, 500), + (800, 600), + (1920, 1080), + ], +) +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_tiling_various_sizes( + image_size: tuple[int, int], + model_id: str, +): + """Test tiling with various image sizes.""" + from PIL import Image + + from vllm.transformers_utils.processors.moondream3 import Moondream3Processor + + processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True) + + width, height = image_size + image = Image.new("RGB", (width, height), color="red") + pixel_values, tiling = processor.preprocess_image(image) + + # Basic shape checks + assert pixel_values.dim() == 4 # [num_crops, C, H, W] + assert pixel_values.shape[1] == 3 # RGB + assert pixel_values.shape[2] == 378 # crop height + assert pixel_values.shape[3] == 378 # crop width + + # Tiling should respect max_crops (12) + assert tiling[0] * tiling[1] <= 12 + + +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_pixel_normalization( + model_id: str, +): + """Test that pixel values are normalized to [-1, 1] range.""" + from PIL import Image + + from vllm.transformers_utils.processors.moondream3 import Moondream3Processor + + processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True) + + # Create test image + image = Image.new("RGB", (378, 378), color="green") + pixel_values, _ = processor.preprocess_image(image) + + # Normalization: (x - 0.5) / 0.5 = 2*x - 1 + # For input [0, 1], output should be [-1, 1] + assert pixel_values.min() >= -1.0 + assert pixel_values.max() <= 1.0 + + +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_chat_template_with_image( + image_assets: ImageTestAssets, + model_id: str, +): + """Test that chat template correctly formats BOS + image + prompt.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + tokenizer = ctx.tokenizer + + # Use the chat template format + prompt = "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501 + mm_data = {"image": [image_assets[0].pil_image]} + + processed_inputs = processor( + prompt, + mm_items=processor.info.parse_mm_data(mm_data), + hf_processor_mm_kwargs={}, + ) + token_ids = processed_inputs["prompt_token_ids"] + + # BOS token (<|endoftext|>) should be token ID 0 + bos_token_id = tokenizer.encode("<|endoftext|>", add_special_tokens=False)[0] + assert bos_token_id == 0 + + # First token should be BOS + assert token_ids[0] == bos_token_id + + +@pytest.mark.parametrize( + "content", + [ + pytest.param( + [ + { + "type": "image_url", + "image_url": {"url": "https://example.invalid/image.png"}, + }, + {"type": "text", "text": "What is in this image?"}, + ], + id="image-first", + ), + pytest.param( + [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "https://example.invalid/image.png"}, + }, + ], + id="text-first", + ), + ], +) +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_chat_template_content_list_uses_moondream_image_prefix( + image_assets: ImageTestAssets, + content: list[dict[str, object]], + model_id: str, +): + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + hf_processor = processor.info.get_hf_processor() + + prompt = hf_processor.tokenizer.apply_chat_template( + [{"role": "user", "content": content}], + chat_template=hf_processor.chat_template, + tokenize=False, + ) + + expected_prompt = ( + "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>" + "What is in this image?<|md_reserved_2|>" + ) + assert prompt == expected_prompt + + processed_inputs = processor( + prompt, + mm_items=processor.info.parse_mm_data({"image": [image_assets[0].pil_image]}), + hf_processor_mm_kwargs={}, + ) + image_placeholders = processed_inputs["mm_placeholders"]["image"] + assert len(image_placeholders) == 1 + assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS + + +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_bos_token_always_first( + image_assets: ImageTestAssets, + model_id: str, +): + """Test that BOS token (ID 0) is always at position 0.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + # Start with BOS token explicitly + prompt = "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>Describe this image.<|md_reserved_2|>" # noqa: E501 + mm_data = {"image": [image_assets[0].pil_image]} + + processed_inputs = processor( + prompt, + mm_items=processor.info.parse_mm_data(mm_data), + hf_processor_mm_kwargs={}, + ) + token_ids = processed_inputs["prompt_token_ids"] + + # Token ID 0 (<|endoftext|>) should be the first token + assert token_ids[0] == 0, ( + f"Expected BOS token (0) at position 0, got {token_ids[0]}" + ) + + +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_processor_with_small_image( + model_id: str, +): + """Test processor with image smaller than crop size.""" + from PIL import Image + + from vllm.transformers_utils.processors.moondream3 import Moondream3Processor + + processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True) + + # Small image (smaller than crop size) + small_image = Image.new("RGB", (100, 100), color="yellow") + pixel_values, tiling = processor.preprocess_image(small_image) + + # Small images should use 1x1 tiling + assert tiling == (1, 1) + # Should have 2 crops (global + 1 local) + assert pixel_values.shape[0] == 2 + + +@pytest.mark.parametrize( + "image_kind", + [ + pytest.param("numpy_hwc", id="numpy-hwc"), + pytest.param("numpy_chw", id="numpy-chw"), + pytest.param("torch_chw", id="torch-chw"), + ], +) +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_preprocess_image_accepts_non_pil_inputs( + image_assets: ImageTestAssets, + image_kind: str, + model_id: str, +): + from vllm.transformers_utils.processors.moondream3 import Moondream3Processor + + processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True) + pil_image = image_assets[0].pil_image.convert("RGB") + hwc_array = np.asarray(pil_image) + expected_pixel_values, expected_tiling = processor.preprocess_image(pil_image) + + if image_kind == "numpy_hwc": + image = hwc_array + elif image_kind == "numpy_chw": + image = np.transpose(hwc_array, (2, 0, 1)) + else: + image = torch.from_numpy(np.transpose(hwc_array, (2, 0, 1)).copy()) + + pixel_values, tiling = processor.preprocess_image(image) + + assert tiling == expected_tiling + assert pixel_values.shape == expected_pixel_values.shape + assert pixel_values.dtype == torch.bfloat16 + assert torch.equal(pixel_values, expected_pixel_values) + + +@pytest.mark.parametrize("image_kind", ["numpy_chw", "torch_chw"]) +@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID]) +def test_processor_apply_accepts_non_pil_image_inputs( + image_assets: ImageTestAssets, + image_kind: str, + model_id: str, +): + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + prompt = "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501 + hwc_array = np.asarray(image_assets[0].pil_image.convert("RGB")) + chw_array = np.transpose(hwc_array, (2, 0, 1)).copy() + image = chw_array if image_kind == "numpy_chw" else torch.from_numpy(chw_array) + + processed_inputs = processor( + prompt, + mm_items=processor.info.parse_mm_data({"image": [image]}), + hf_processor_mm_kwargs={}, + ) + + image_placeholders = processed_inputs["mm_placeholders"]["image"] + assert len(image_placeholders) == 1 + assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS + + mm_kwargs = processed_inputs["mm_kwargs"].get_data() + assert mm_kwargs["pixel_values"].shape[2:] == (3, 378, 378) + + +class TestMoondream3TilingLogic: + """CPU-based tests for Moondream3 tiling selection logic. + + These tests validate the select_tiling() function which determines + how images are divided into crops for the vision encoder. + """ + + def test_small_image_no_tiling(self): + """Small images should use 1x1 tiling.""" + from vllm.transformers_utils.processors.moondream3 import select_tiling + + tiling = select_tiling( + height=300, width=300, crop_size=CROP_SIZE, max_crops=MAX_CROPS + ) + assert tiling == (1, 1) + + def test_exact_crop_size(self): + """Image exactly at crop size should use 1x1.""" + from vllm.transformers_utils.processors.moondream3 import select_tiling + + tiling = select_tiling( + height=CROP_SIZE, width=CROP_SIZE, crop_size=CROP_SIZE, max_crops=MAX_CROPS + ) + assert tiling == (1, 1) + + def test_large_square_image(self): + """Large square image should use multiple tiles.""" + from vllm.transformers_utils.processors.moondream3 import select_tiling + + tiling = select_tiling( + height=800, width=800, crop_size=CROP_SIZE, max_crops=MAX_CROPS + ) + h_tiles, w_tiles = tiling + assert h_tiles >= 2 + assert w_tiles >= 2 + assert h_tiles * w_tiles <= MAX_CROPS + + def test_wide_image(self): + """Wide image should have more width tiles.""" + from vllm.transformers_utils.processors.moondream3 import select_tiling + + tiling = select_tiling( + height=400, width=1200, crop_size=CROP_SIZE, max_crops=MAX_CROPS + ) + h_tiles, w_tiles = tiling + assert w_tiles >= h_tiles + + def test_tall_image(self): + """Tall image should have more height tiles.""" + from vllm.transformers_utils.processors.moondream3 import select_tiling + + tiling = select_tiling( + height=1200, width=400, crop_size=CROP_SIZE, max_crops=MAX_CROPS + ) + h_tiles, w_tiles = tiling + assert h_tiles >= w_tiles + + def test_respects_max_crops(self): + """Tiling should not exceed max_crops.""" + from vllm.transformers_utils.processors.moondream3 import select_tiling + + tiling = select_tiling( + height=2000, width=2000, crop_size=CROP_SIZE, max_crops=4 + ) + h_tiles, w_tiles = tiling + assert h_tiles * w_tiles <= 4 + + +class TestMoondream3VisionShapes: + """CPU-based tests for vision encoder expected shapes. + + These tests verify the mathematical relationships between + crop size, patch size, and token counts. + """ + + def test_expected_patch_count(self): + """Test 378/14 = 27 patches per side, 729 total.""" + patches_per_side = CROP_SIZE // PATCH_SIZE + total_patches = patches_per_side**2 + + assert patches_per_side == 27 + assert total_patches == EXPECTED_IMAGE_TOKENS - 1 + + def test_patch_embedding_input_dim(self): + """Test patch embedding input dimension.""" + channels = 3 + input_dim = PATCH_SIZE * PATCH_SIZE * channels + + assert input_dim == 14 * 14 * 3 + assert input_dim == 588 + + +class TestMoondream3TauAttention: + """CPU-based tests for tau attention scaling components. + + These tests validate the tau attention formula used in Moondream3: + - Token-based: tok_q = tanh(gelu(qkv) @ tau_wq.T) + - Position-based: tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5) + """ + + def test_tau_position_range(self): + """Test tau position scaling produces values in valid range.""" + num_heads = 32 + seq_len = 100 + + tau_alpha = torch.randn(num_heads) + positions = torch.arange(seq_len) + + pos_float = (positions.float() + 1.0).clamp(min=1e-6) + pos_log = pos_float.log() + tau_pos = 1.0 + (torch.sigmoid(tau_alpha[:, None] * pos_log[None, :]) - 0.5) + + assert tau_pos.shape == (num_heads, seq_len) + # tau_pos should be between 0.5 and 1.5 + assert tau_pos.min() >= 0.5 + assert tau_pos.max() <= 1.5 + + def test_tau_token_output_range(self): + """Test tau token scaling output is bounded by tanh.""" + import torch.nn.functional as F + + seq_len = 100 + qkv_dim = 6144 # 2048 * 3 + num_heads = 32 + + qkv = torch.randn(seq_len, qkv_dim) + tau_wq = torch.randn(num_heads, qkv_dim) + + tok_feat = F.gelu(qkv) + tok_q = torch.tanh(tok_feat @ tau_wq.t()) + + assert tok_q.shape == (seq_len, num_heads) + # tanh output is bounded by [-1, 1] + assert tok_q.min() >= -1.0 + assert tok_q.max() <= 1.0 diff --git a/tests/models/registry.py b/tests/models/registry.py index 84019eabec..21d3a50ce9 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1122,6 +1122,16 @@ _MULTIMODAL_EXAMPLE_MODELS = { extras={"olmo": "allenai/Molmo-7B-O-0924"}, trust_remote_code=True, ), + "Moondream3ForCausalLM": _HfExamplesInfo( + "moondream/moondream3-preview", + tokenizer="moondream/starmie-v1", + trust_remote_code=True, + ), + "HfMoondream": _HfExamplesInfo( + "moondream/moondream3-preview", + tokenizer="moondream/starmie-v1", + trust_remote_code=True, + ), "Molmo2ForConditionalGeneration": _HfExamplesInfo( "allenai/Molmo2-8B", extras={"olmo": "allenai/Molmo2-O-7B"}, diff --git a/tests/models/utils.py b/tests/models/utils.py index b93beee6aa..b12ab72d77 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -523,6 +523,17 @@ def dummy_hf_overrides( text_config.update(update_dict) + # Update n_layers and moe configs for Moondream3 model + if model_arch in ("Moondream3ForCausalLM", "HfMoondream"): + text_config.update( + { + "n_layers": num_hidden_layers, + "moe_num_experts": num_experts, + "moe_experts_per_token": 2, + "moe_start_layer": num_hidden_layers, + } + ) + if hasattr(hf_config, "vision_config"): hf_config.vision_config.update( { @@ -531,6 +542,9 @@ def dummy_hf_overrides( } ) + if model_arch in ("Moondream3ForCausalLM", "HfMoondream"): + hf_config.vision_config.update({"enc_n_layers": 1}) + # e.g.: ibm-granite/granite-speech-3.3-2b if hasattr(hf_config, "encoder_config"): hf_config.encoder_config.update( diff --git a/vllm/model_executor/models/moondream3.py b/vllm/model_executor/models/moondream3.py new file mode 100644 index 0000000000..d5f3e6b195 --- /dev/null +++ b/vllm/model_executor/models/moondream3.py @@ -0,0 +1,1423 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only Moondream3 model implementation.""" + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from functools import cached_property +from itertools import islice + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import BatchFeature + +from vllm.config import VllmConfig +from vllm.config.multimodal import BaseDummyOptions +from vllm.distributed import ( + get_pp_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_gather, + tensor_model_parallel_all_reduce, +) +from vllm.inputs import MultiModalDataDict +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.layers.attention.mm_encoder_attention import ( + MMEncoderAttention, +) +from vllm.model_executor.layers.fused_moe import MoEActivation, fused_experts +from vllm.model_executor.layers.fused_moe.config import biased_moe_quant_config +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + QKVParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import ( + MultiModalFieldConfig, + MultiModalKwargsItems, +) +from vllm.multimodal.parse import ImageSize, MultiModalDataItems +from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, + BaseMultiModalProcessor, + BaseProcessingInfo, + PromptReplacement, + PromptUpdate, + PromptUpdateDetails, +) +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.moondream3 import ( + Moondream3Config, + Moondream3TextConfig, + Moondream3VisionConfig, +) +from vllm.transformers_utils.processors.moondream3 import Moondream3Processor + +from .interfaces import ( + MultiModalEmbeddings, + SupportsMultiModal, + SupportsPP, +) +from .utils import ( + extract_layer_index, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) + +logger = init_logger(__name__) + + +# ============================================================================ +# Image Processing Utilities +# ============================================================================ + + +def reconstruct_from_crops( + crops: torch.Tensor, + tiling: tuple[int, int], + overlap_margin: int, + patch_size: int = 14, +) -> torch.Tensor: + """Reconstruct features from overlapping crops.""" + tiling_h, tiling_w = tiling + crop_height, crop_width = crops[0].shape[:2] + margin_pixels = overlap_margin * patch_size + + output_h = (crop_height - 2 * margin_pixels) * tiling_h + 2 * margin_pixels + output_w = (crop_width - 2 * margin_pixels) * tiling_w + 2 * margin_pixels + + reconstructed = torch.zeros( + (output_h, output_w, crops[0].shape[2]), + device=crops[0].device, + dtype=crops[0].dtype, + ) + + for i, crop in enumerate(crops): + tile_y = i // tiling_w + tile_x = i % tiling_w + + x_start = 0 if tile_x == 0 else margin_pixels + x_end = crop_width if tile_x == tiling_w - 1 else crop_width - margin_pixels + y_start = 0 if tile_y == 0 else margin_pixels + y_end = crop_height if tile_y == tiling_h - 1 else crop_height - margin_pixels + + out_x = tile_x * (crop_width - 2 * margin_pixels) + out_y = tile_y * (crop_height - 2 * margin_pixels) + + reconstructed[ + out_y + y_start : out_y + y_end, out_x + x_start : out_x + x_end + ] = crop[y_start:y_end, x_start:x_end] + + return reconstructed + + +# ============================================================================ +# Vision Encoder Components +# ============================================================================ + + +class Moondream3VisionMLP(nn.Module): + """MLP for vision encoder blocks.""" + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.fc1 = ColumnParallelLinear( + hidden_size, + intermediate_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.fc1", + ) + self.act = get_act_fn("gelu_pytorch_tanh") + self.fc2 = RowParallelLinear( + intermediate_size, + hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.fc2", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, _ = self.fc1(x) + x = self.act(x) + x, _ = self.fc2(x) + return x + + +class Moondream3VisionAttention(nn.Module): + """Self-attention for vision encoder (bidirectional).""" + + def __init__( + self, + hidden_size: int, + num_heads: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.hidden_size = hidden_size + self.num_heads = num_heads + self.head_dim = hidden_size // num_heads + + self.qkv_proj = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=num_heads, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.out_proj = RowParallelLinear( + input_size=hidden_size, + output_size=hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.out_proj", + ) + + tp_size = get_tensor_model_parallel_world_size() + self.num_heads_per_partition = num_heads // tp_size + + self.attn = MMEncoderAttention( + num_heads=self.num_heads_per_partition, + head_size=self.head_dim, + scale=self.head_dim**-0.5, + prefix=f"{prefix}.attn", + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.chunk(3, dim=-1) + out = self.attn(q, k, v) + out, _ = self.out_proj(out) + return out + + +class Moondream3VisionBlock(nn.Module): + """Transformer block for vision encoder.""" + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + num_heads: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.ln1 = nn.LayerNorm(hidden_size, eps=1e-5) + self.attn = Moondream3VisionAttention( + hidden_size=hidden_size, + num_heads=num_heads, + quant_config=quant_config, + prefix=f"{prefix}.attn", + ) + self.ln2 = nn.LayerNorm(hidden_size, eps=1e-5) + self.mlp = Moondream3VisionMLP( + hidden_size=hidden_size, + intermediate_size=intermediate_size, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.ln1(x)) + x = x + self.mlp(self.ln2(x)) + return x + + +class Moondream3VisionEncoder(nn.Module): + """Vision encoder (SigLIP-style ViT).""" + + def __init__( + self, + config: Moondream3VisionConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.config = config + + # Patch embedding + self.patch_emb = nn.Linear( + config.enc_patch_size * config.enc_patch_size * 3, + config.enc_dim, + bias=True, + ) + + # Position embeddings (27x27 = 729 patches for 378x378 / 14) + num_patches = (config.crop_size // config.enc_patch_size) ** 2 + self.pos_emb = nn.Parameter(torch.zeros(1, num_patches, config.enc_dim)) + + # Transformer blocks + self.blocks = nn.ModuleList( + [ + Moondream3VisionBlock( + hidden_size=config.enc_dim, + intermediate_size=config.enc_ff_dim, + num_heads=config.enc_n_heads, + quant_config=quant_config, + prefix=f"{prefix}.blocks.{i}", + ) + for i in range(config.enc_n_layers) + ] + ) + + self.post_ln = nn.LayerNorm(config.enc_dim, eps=1e-5) + + def create_patches(self, images: torch.Tensor) -> torch.Tensor: + """Convert images to patch embeddings. + + Args: + images: (batch, channels, height, width) + + Returns: + patches: (batch, num_patches, patch_dim) + """ + patch_size = self.config.enc_patch_size + batch, channels, height, width = images.shape + patches_h = height // patch_size + patches_w = width // patch_size + + # Unfold into patches + patches = images.unfold(2, patch_size, patch_size).unfold( + 3, patch_size, patch_size + ) + # (batch, channels, patches_h, patches_w, patch_size, patch_size) + patches = patches.permute(0, 2, 3, 1, 4, 5).contiguous() + # (batch, patches_h, patches_w, channels, patch_size, patch_size) + patches = patches.view(batch, patches_h * patches_w, -1) + # (batch, num_patches, channels * patch_size * patch_size) + + return patches + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + """Encode images. + + Args: + pixel_values: (batch, channels, height, width) + + Returns: + features: (batch, num_patches, hidden_size) + """ + # Create patches and embed + patches = self.create_patches(pixel_values) + x = self.patch_emb(patches) + + # Add position embeddings + x = x + self.pos_emb + + # Apply transformer blocks + for block in self.blocks: + x = block(x) + + # Final layer norm + x = self.post_ln(x) + + return x + + +class Moondream3VisionProjection(nn.Module): + """Projects vision features to text embedding dimension.""" + + def __init__( + self, + input_dim: int, + inner_dim: int, + output_dim: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + # Input is concatenated global and local features (2 * input_dim) + self.fc1 = ColumnParallelLinear( + input_dim * 2, + inner_dim, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.fc1", + ) + self.act = get_act_fn("gelu_pytorch_tanh") + self.fc2 = RowParallelLinear( + inner_dim, + output_dim, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.fc2", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, _ = self.fc1(x) + x = self.act(x) + x, _ = self.fc2(x) + return x + + +# ============================================================================ +# Text Decoder Components +# ============================================================================ + + +class Moondream3TextMLP(nn.Module): + """Standard MLP for non-MoE layers (layers 0-3).""" + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.fc1 = ColumnParallelLinear( + hidden_size, + intermediate_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.fc1", + ) + self.act = get_act_fn("gelu_pytorch_tanh") + self.fc2 = RowParallelLinear( + intermediate_size, + hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.fc2", + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, _ = self.fc1(x) + x = self.act(x) + x, _ = self.fc2(x) + return x + + +class Moondream3TextMoE(nn.Module): + """Mixture of Experts layer for layers 4+ with expert parallelism. + + Moondream3 uses a custom GeGLU activation: gelu(h) * (g + 1) + where fc1 outputs [gate, up] and the activation is gelu(gate) * (up + 1). + + Uses expert parallelism where each GPU stores num_experts/tp_size experts. + Routing and communication handled via all-to-all or replicated computation. + + Checkpoint format: + - fc1.weight: [num_experts, expert_inner_dim * 2, hidden_size] (gate+up) + - fc2.weight: [num_experts, hidden_size, expert_inner_dim] (down) + - router.weight: [num_experts, hidden_size] + - router.bias: [num_experts] + """ + + def __init__( + self, + hidden_size: int, + expert_inner_dim: int, + num_experts: int, + experts_per_token: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.hidden_size = hidden_size + self.expert_inner_dim = expert_inner_dim + self.num_experts = num_experts + self.experts_per_token = experts_per_token + + # Expert parallelism: each GPU stores a subset of experts + self.tp_size = get_tensor_model_parallel_world_size() + self.experts_per_rank = num_experts // self.tp_size + self.num_local_experts = self.experts_per_rank + + # Router (gate) - use ReplicatedLinear for compatibility + self.gate = ReplicatedLinear( + hidden_size, + num_experts, + bias=True, + quant_config=None, + prefix=f"{prefix}.gate", + ) + + # Local expert weights (only store experts_per_rank experts) + # fc1: [experts_per_rank, expert_inner_dim * 2, hidden_size] + # fc2: [experts_per_rank, hidden_size, expert_inner_dim] + self.fc1_weight = nn.Parameter( + torch.empty(self.num_local_experts, expert_inner_dim * 2, hidden_size) + ) + self.fc2_weight = nn.Parameter( + torch.empty(self.num_local_experts, hidden_size, expert_inner_dim) + ) + self._use_fused_moe = True + + local_expert_start = get_tensor_model_parallel_rank() * self.experts_per_rank + expert_map = torch.full((num_experts,), -1, dtype=torch.int32) + expert_map[local_expert_start : local_expert_start + self.num_local_experts] = ( + torch.arange(self.num_local_experts, dtype=torch.int32) + ) + self.register_buffer("_expert_map", expert_map, persistent=False) + + # Preserve Moondream3's exact GeGLU variant (gelu(h) * (g + 1)) by + # adding +1 bias to the second half of the fused fc1 activations. + fused_w1_bias = torch.zeros( + self.num_local_experts, + expert_inner_dim * 2, + dtype=torch.float32, + ) + fused_w1_bias[:, expert_inner_dim:] = 1.0 + self.register_buffer("_fused_w1_bias", fused_w1_bias, persistent=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward pass with expert parallelism and custom GeGLU activation.""" + + # Get router logits and compute top-k + router_logits, _ = self.gate(x) # [num_tokens, num_experts] + topk_logits, topk_ids = torch.topk( + router_logits, self.experts_per_token, dim=-1 + ) + # Softmax over selected experts + topk_weights = F.softmax(topk_logits, dim=-1, dtype=torch.float32).to(x.dtype) + + if self._use_fused_moe and x.is_cuda: + try: + out = fused_experts( + hidden_states=x.contiguous(), + w1=self.fc1_weight, + w2=self.fc2_weight, + topk_weights=topk_weights.contiguous(), + topk_ids=topk_ids.contiguous(), + activation=MoEActivation.GELU, + global_num_experts=self.num_experts, + expert_map=self._expert_map, + quant_config=biased_moe_quant_config(self._fused_w1_bias, None), + ) + out = tensor_model_parallel_all_reduce(out) + return out + except (NotImplementedError, RuntimeError) as exc: + self._use_fused_moe = False + logger.warning_once( + "Disabling fused Moondream3 MoE path and falling back to " + "the Python expert loop: %s", + str(exc), + ) + + tp_rank = get_tensor_model_parallel_rank() + # Compute local expert range + local_expert_start = tp_rank * self.experts_per_rank + + # Fallback path for environments where fused kernels are unavailable. + out = x.new_zeros(x.shape) + + for local_expert_idx in range(self.num_local_experts): + global_expert_id = local_expert_start + local_expert_idx + + # Find tokens assigned to this expert + token_pos, which_k = (topk_ids == global_expert_id).nonzero(as_tuple=True) + if token_pos.numel() == 0: + continue + + # Get tokens and their routing weights + x_tok = x.index_select(0, token_pos) # [n_tokens, hidden_size] + gate_tok = topk_weights[token_pos, which_k] # [n_tokens] + + # fc1: [expert_inner_dim * 2, hidden_size] + # h_full: [n_tokens, expert_inner_dim * 2] + h_full = F.linear(x_tok, self.fc1_weight[local_expert_idx]) + + # GeGLU with (g + 1): h, g = split; output = gelu(h) * (g + 1) + # HF MoE uses exact GELU (not tanh approximation). + h, g = h_full.chunk(2, dim=-1) # Each [n_tokens, expert_inner_dim] + h = F.gelu(h) * (g + 1.0) + + # fc2: [hidden_size, expert_inner_dim] + # y: [n_tokens, hidden_size] + y = F.linear(h, self.fc2_weight[local_expert_idx]) + + # Apply routing weight + y = y * gate_tok.unsqueeze(-1) + + # Accumulate output + out.index_add_(0, token_pos, y) + + # All-reduce to combine results from all experts across GPUs + out = tensor_model_parallel_all_reduce(out) + + return out + + +class Moondream3Attention(nn.Module): + """Decoder attention with RoPE and tau scaling. + + Moondream3 uses a tau attention mechanism that scales Q and V + based on both token content and position. + """ + + def __init__( + self, + config: Moondream3TextConfig, + layer_idx: int, + cache_config=None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.hidden_size = config.dim + self.num_heads = config.n_heads + self.num_kv_heads = config.n_kv_heads + self.head_dim = config.dim // config.n_heads + + tp_size = get_tensor_model_parallel_world_size() + self.num_heads_per_partition = self.num_heads // tp_size + self.num_kv_heads_per_partition = max(1, self.num_kv_heads // tp_size) + + self.qkv_proj = QKVParallelLinear( + hidden_size=self.hidden_size, + head_size=self.head_dim, + total_num_heads=self.num_heads, + total_num_kv_heads=self.num_kv_heads, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + + self.out_proj = RowParallelLinear( + input_size=self.hidden_size, + output_size=self.hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.out_proj", + ) + + # Moondream uses 32-dim rotation out of 64-dim head (partial_rotary_factor=0.5) + # HF Moondream uses non-interleaved RoPE (split by half) + # In vLLM, is_neox_style=True means split by half (GPT-NeoX style) + rope_parameters = { + "rope_theta": config.rope_theta, + "partial_rotary_factor": 32 / self.head_dim, # 32/64 = 0.5 + } + self.rotary_emb = get_rope( + head_size=self.head_dim, + max_position=config.max_context, + rope_parameters=rope_parameters, + is_neox_style=True, # Moondream uses split-by-half (GPT-NeoX) style + ) + + self.scaling = self.head_dim**-0.5 + self.attn = Attention( + num_heads=self.num_heads_per_partition, + head_size=self.head_dim, + scale=self.scaling, + num_kv_heads=self.num_kv_heads_per_partition, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn", + ) + + # Tau scaling parameters for position-dependent attention + # These are learned during training to modulate attention based on position + # tau_wq and tau_wv need full qkv_dim for correct computation + # Only heads are partitioned, qkv dimension is kept full for all-gather + qkv_dim = self.hidden_size * 3 # Q + K + V dimension (full) + self.tau_alpha = nn.Parameter(torch.zeros(self.num_heads_per_partition)) + self.tau_wq = nn.Parameter(torch.zeros(self.num_heads_per_partition, qkv_dim)) + self.tau_wv = nn.Parameter(torch.zeros(self.num_heads_per_partition, qkv_dim)) + self.tp_size = tp_size + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + + q, k, v = qkv.split( + [ + self.num_heads_per_partition * self.head_dim, + self.num_kv_heads_per_partition * self.head_dim, + self.num_kv_heads_per_partition * self.head_dim, + ], + dim=-1, + ) + + # Apply tau scaling to Q and V + # Tau scaling has two components: + # 1. Token-based: tok_q = tanh(gelu(qkv) @ tau_wq.T) + # 2. Position-based: tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5) + # Final: tau = tok + tau_pos + # + # For TP, tau weights are sharded by head, but qkv_dim is kept full + + # Get full qkv for tau computation + # With TP, reconstruct qkv in correct layout [q_full, k_full, v_full] + # (all-gather would produce [q_0, k_0, v_0, q_1, k_1, v_1] - wrong) + if self.tp_size > 1: + # All-gather once, then reconstruct [q_full, k_full, v_full]. + qkv_full_sharded = tensor_model_parallel_all_gather(qkv.contiguous()) + q_local_dim = q.shape[-1] + kv_local_dim = k.shape[-1] + qkv_full_sharded = qkv_full_sharded.view( + qkv.shape[0], + self.tp_size, + q_local_dim + 2 * kv_local_dim, + ) + q_full = qkv_full_sharded[:, :, :q_local_dim].reshape(qkv.shape[0], -1) + k_full = qkv_full_sharded[ + :, :, q_local_dim : q_local_dim + kv_local_dim + ].reshape(qkv.shape[0], -1) + v_full = qkv_full_sharded[:, :, q_local_dim + kv_local_dim :].reshape( + qkv.shape[0], -1 + ) + qkv_full = torch.cat([q_full, k_full, v_full], dim=-1).contiguous() + else: + qkv_full = qkv + + # Compute tau scaling factors matching HF implementation exactly: + # tok_feat = gelu(qkv) + # tok_q = tanh(tok_feat @ tau_wq.T) # [num_tokens, num_heads] + # tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5) # [num_heads, num_tokens] + # tau = (tok_q.T + tau_pos).T # [num_tokens, num_heads] + num_tokens = qkv_full.shape[0] + orig_dtype = q.dtype + + # Token-based component + tok_feat = F.gelu(qkv_full) # Apply GELU activation + tok_q = torch.tanh(tok_feat @ self.tau_wq.t()) # [N, H_per_partition] + tok_v = torch.tanh(tok_feat @ self.tau_wv.t()) # [N, H_per_partition] + + # Position-based component + # tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5) + # positions is [num_tokens], need to compute for each head + # tau_alpha: [num_heads_per_partition] + pos_float = (positions.to(orig_dtype) + 1.0).clamp(min=1e-6) + pos_log = pos_float.log() # [num_tokens] + # alpha[:, None] * pos_log[None, :] -> [num_heads, num_tokens] + tau_pos = 1.0 + ( + torch.sigmoid(self.tau_alpha[:, None] * pos_log[None, :]) - 0.5 + ) # [H_per_partition, N] + + # Combine token and position components + tau_q = (tok_q + tau_pos.t()).to(orig_dtype) # [N, H_per_partition] + tau_v = (tok_v + tau_pos.t()).to(orig_dtype) # [N, H_per_partition] + + # Reshape q and v to apply per-head tau scaling + q = q.view(num_tokens, self.num_heads_per_partition, self.head_dim) + v = v.view(num_tokens, self.num_kv_heads_per_partition, self.head_dim) + + # Apply tau scaling + q = q * tau_q.unsqueeze(-1) + v = v * tau_v[:, : self.num_kv_heads_per_partition].unsqueeze(-1) + + # Reshape back + q = q.view(num_tokens, -1) + v = v.view(num_tokens, -1) + + q, k = self.rotary_emb(positions, q, k) + + attn_output = self.attn(q, k, v) + + output, _ = self.out_proj(attn_output) + return output + + +class Moondream3DecoderLayer(nn.Module): + """Decoder layer with attention + MLP/MoE.""" + + def __init__( + self, + config: Moondream3TextConfig, + cache_config=None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + layer_idx = extract_layer_index(prefix) + self.layer_idx = layer_idx + + self.ln = nn.LayerNorm(config.dim, eps=1e-5, bias=True) + + self.attn = Moondream3Attention( + config=config, + layer_idx=layer_idx, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn", + ) + + # Use MoE for layers >= moe_start_layer, standard MLP otherwise + if layer_idx >= config.moe_start_layer: + self.mlp = Moondream3TextMoE( + hidden_size=config.dim, + expert_inner_dim=config.moe_expert_inner_dim, + num_experts=config.moe_num_experts, + experts_per_token=config.moe_experts_per_token, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + else: + self.mlp = Moondream3TextMLP( + hidden_size=config.dim, + intermediate_size=config.ff_dim, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + # Pre-norm architecture + normed = self.ln(hidden_states) + attn_out = self.attn(positions, normed) + mlp_out = self.mlp(normed) + hidden_states = hidden_states + attn_out + mlp_out + return hidden_states + + +class Moondream3TextModel(nn.Module): + """Text decoder model.""" + + def __init__( + self, + config: Moondream3TextConfig, + cache_config=None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): + super().__init__() + self.config = config + + self.wte = VocabParallelEmbedding( + config.vocab_size, + config.dim, + prefix=f"{prefix}.wte", + ) + + blocks_prefix = maybe_prefix(prefix, "blocks") + self.start_layer, self.end_layer, self.blocks = make_layers( + config.n_layers, + lambda prefix: Moondream3DecoderLayer( + config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + ), + prefix=blocks_prefix, + ) + + self.post_ln = nn.LayerNorm(config.dim, eps=1e-5, bias=True) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states"], config.dim + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.wte(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor | IntermediateTensors: + pp_group = get_pp_group() + if pp_group.is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + assert input_ids is not None + hidden_states = self.embed_input_ids(input_ids) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + + for i, layer in enumerate( + islice(self.blocks, self.start_layer, self.end_layer) + ): + hidden_states = layer(positions, hidden_states) + + if not pp_group.is_last_rank: + return IntermediateTensors({"hidden_states": hidden_states}) + + hidden_states = self.post_ln(hidden_states) + return hidden_states + + +@dataclass(frozen=True) +class Moondream3ImageInput: + """Container holding per-image inputs for embedding.""" + + pixel_values: torch.Tensor + tiling: tuple[int, int] | None + + +# ============================================================================ +# Multimodal Processing +# ============================================================================ + + +class Moondream3ProcessingInfo(BaseProcessingInfo): + """Processing info for Moondream3.""" + + def get_hf_config(self): + return self.ctx.get_hf_config() + + def get_hf_processor(self, **kwargs: object): + return self.ctx.get_hf_processor(Moondream3Processor, **kwargs) + + def get_supported_mm_limits(self) -> Mapping[str, int | None]: + return {"image": 1} + + def get_num_image_tokens( + self, + *, + image_width: int, + image_height: int, + ) -> int: + # HF pre-fills BOS together with the fixed 27x27 vision grid under + # the same bidirectional prefix mask: 1 BOS + 729 image embeddings. + return 730 + + def get_image_size_with_most_features(self) -> ImageSize: + return ImageSize(width=378, height=378) + + def get_max_image_tokens(self) -> int: + return 730 + + def get_mm_max_tokens_per_item( + self, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> Mapping[str, int]: + return {"image": self.get_max_image_tokens()} + + +class Moondream3DummyInputsBuilder(BaseDummyInputsBuilder[Moondream3ProcessingInfo]): + """Dummy inputs builder for profiling.""" + + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: + return ( + "<|endoftext|><|md_reserved_0|>query<|md_reserved_1|>" + "What is this image?<|md_reserved_2|>" + ) + + def get_dummy_mm_data( + self, + seq_len: int, + mm_counts: Mapping[str, int], + mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_processor_kwargs: Mapping[str, object] | None = None, + ) -> MultiModalDataDict: + num_images = mm_counts.get("image", 0) + return { + "image": self._get_dummy_images( + width=378, + height=378, + num_images=num_images, + ) + } + + +class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3ProcessingInfo]): + """Multimodal processor for Moondream3.""" + + image_placeholder: str = "" + bos_image_placeholder: str = "<|endoftext|>" + + def _call_hf_processor( + self, + prompt: str, + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], + tok_kwargs: Mapping[str, object], + ) -> BatchFeature: + # Moondream3's processor handles images directly rather than exposing a + # separate `image_processor`, so keep the cache path on text+MM calls. + return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) + + @cached_property + def bos_image_placeholder_tokens(self) -> list[int]: + tokenizer = self.info.get_tokenizer() + token_ids = tokenizer.encode( + self.bos_image_placeholder, + add_special_tokens=False, + ) + if len(token_ids) < 2: + raise ValueError( + "Tokenizer could not encode Moondream3 BOS/image placeholder " + f"{self.bos_image_placeholder!r}." + ) + return token_ids + + def _get_mm_fields_config( + self, + hf_inputs: BatchFeature, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> Mapping[str, MultiModalFieldConfig]: + return { + "pixel_values": MultiModalFieldConfig.batched("image"), + "tilings": MultiModalFieldConfig.batched("image", keep_on_cpu=True), + } + + def _hf_processor_applies_updates( + self, + prompt_text: str, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + tokenization_kwargs: Mapping[str, object], + ) -> bool: + # Moondream3 HF processor does NOT expand placeholder tokens. + # vLLM expands BOS + so the whole HF image prefix is marked + # bidirectional by the multimodal prefix-LM mask. + return False + + def _get_prompt_updates( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + out_mm_kwargs: MultiModalKwargsItems, + ) -> list[PromptUpdate]: + image_size = self.info.get_image_size_with_most_features() + num_image_tokens = self.info.get_num_image_tokens( + image_width=image_size.width, + image_height=image_size.height, + ) + placeholder_tokens = self.bos_image_placeholder_tokens + bos_token = placeholder_tokens[0] + image_token = placeholder_tokens[-1] + return [ + PromptReplacement( + modality="image", + target=placeholder_tokens, + replacement=PromptUpdateDetails( + full=[bos_token] + [image_token] * (num_image_tokens - 1), + ), + ), + ] + + +# ============================================================================ +# Main Model +# ============================================================================ + + +@MULTIMODAL_REGISTRY.register_processor( + Moondream3MultiModalProcessor, + info=Moondream3ProcessingInfo, + dummy_inputs=Moondream3DummyInputsBuilder, +) +class Moondream3ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP): + """Moondream3 multimodal model for causal language modeling. + + vLLM supports the standard autoregressive Moondream3 query and caption + prompt formats. The region-module point/detect skills require custom + coordinate decoding and are intentionally not exposed here. + """ + + supports_multimodal = True + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + } + + def __init__( + self, + *, + vllm_config: VllmConfig, + prefix: str = "", + ): + super().__init__() + + hf_config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + cache_config = vllm_config.cache_config + + # Reuse the transformers_utils config implementation. + if isinstance(hf_config, Moondream3Config): + self.config = hf_config + else: + config_dict = hf_config.config if hasattr(hf_config, "config") else {} + self.config = Moondream3Config(config=config_dict) + + with self._mark_tower_model(vllm_config, "image"): + # Vision encoder + self.vision = Moondream3VisionEncoder( + config=self.config.vision_config, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "vision"), + ) + + # Vision projection + self.vision_proj = Moondream3VisionProjection( + input_dim=self.config.vision_config.enc_dim, + inner_dim=self.config.vision_config.proj_inner_dim, + output_dim=self.config.text_config.dim, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "vision_proj"), + ) + + with self._mark_language_model(vllm_config): + # Text decoder + self.text = Moondream3TextModel( + config=self.config.text_config, + cache_config=cache_config, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "text"), + ) + + # LM head (with bias - Moondream3 has lm_head bias) + self.lm_head = ParallelLMHead( + self.config.text_config.vocab_size, + self.config.text_config.dim, + bias=True, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + + self.logits_processor = LogitsProcessor(self.config.text_config.vocab_size) + self.make_empty_intermediate_tensors = self.text.make_empty_intermediate_tensors + self._answer_id = getattr( + self.config, + "answer_token_id", + getattr(hf_config, "answer_token_id", 3), + ) + + @classmethod + def get_placeholder_str(cls, modality: str, i: int) -> str | None: + if modality == "image": + return "" + return None + + def get_language_model(self) -> nn.Module: + return self.text + + def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: + return num_image_tokens + + def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: + return num_vision_tokens + + def _split_pixel_values( + self, + pixel_values: object, + ) -> list[torch.Tensor]: + # The processor should standardize image inputs into: + # - torch.Tensor [num_images, num_crops, C, H, W], or + # - list[torch.Tensor[num_crops, C, H, W]] for ragged crops. + if isinstance(pixel_values, torch.Tensor): + if pixel_values.dim() != 5: + raise ValueError( + "Expected `pixel_values` tensor with shape " + "[num_images, num_crops, C, H, W], got " + f"{tuple(pixel_values.shape)}." + ) + return [pv.contiguous() for pv in pixel_values] + + if isinstance(pixel_values, (list, tuple)): + tensors: list[torch.Tensor] = [] + for value in pixel_values: + if not isinstance(value, torch.Tensor): + raise TypeError( + "Expected each `pixel_values` element to be a tensor, " + f"got {type(value)!r}." + ) + if value.dim() != 4: + raise ValueError( + f"Unsupported pixel_values element shape {tuple(value.shape)}." + ) + tensors.append(value.contiguous()) + return tensors + + raise TypeError( + "pixel_values must be a tensor or a sequence of tensors, " + f"got {type(pixel_values)!r}." + ) + + def _split_tilings( + self, + tilings: object, + expected: int, + ) -> list[tuple[int, int] | None]: + if tilings is None: + return [None] * expected + + if isinstance(tilings, torch.Tensor): + if tilings.dim() != 2 or tilings.shape[1] != 2: + raise ValueError( + "Expected `tilings` tensor with shape [num_images, 2], got " + f"{tuple(tilings.shape)}." + ) + tiling_items = tilings.tolist() + elif isinstance(tilings, (list, tuple)): + tiling_items = list(tilings) + else: + raise TypeError( + "tilings must be None, a tensor or a sequence of tuples, " + f"got {type(tilings)!r}." + ) + + if len(tiling_items) != expected: + raise ValueError( + "Mismatch between the number of pixel_values entries " + f"({expected}) and tilings ({len(tiling_items)})." + ) + + normalized: list[tuple[int, int] | None] = [] + for tiling in tiling_items: + if tiling is None: + normalized.append(None) + continue + if isinstance(tiling, torch.Tensor): + tiling = tiling.tolist() + if isinstance(tiling, (list, tuple)) and len(tiling) == 2: + normalized.append((int(tiling[0]), int(tiling[1]))) + else: + raise ValueError( + f"Each tiling entry must be a pair of integers, got {tiling!r}." + ) + return normalized + + def _parse_image_inputs(self, **kwargs: object) -> list[Moondream3ImageInput]: + pixel_values = kwargs.get("pixel_values") + if pixel_values is None: + return [] + + pixel_values_list = self._split_pixel_values(pixel_values) + tilings_list = self._split_tilings( + kwargs.get("tilings"), len(pixel_values_list) + ) + + image_inputs: list[Moondream3ImageInput] = [] + for value, tiling in zip(pixel_values_list, tilings_list): + if value.dim() != 4: + raise ValueError( + f"Expected 4D tensor for crops, got {tuple(value.shape)}." + ) + image_inputs.append(Moondream3ImageInput(pixel_values=value, tiling=tiling)) + return image_inputs + + def _encode_image_input(self, image_input: Moondream3ImageInput) -> torch.Tensor: + pixel_values = image_input.pixel_values + if pixel_values.dim() != 4: + raise ValueError( + f"Expected 4D tensor for crops, got {tuple(pixel_values.shape)}." + ) + + device = self.vision.patch_emb.weight.device + dtype = self.vision.patch_emb.weight.dtype + pixel_values = pixel_values.to(device=device, dtype=dtype) + + features = self.vision(pixel_values) + + # Grid size = crop_size / patch_size (e.g., 378 / 14 = 27) + grid_size = ( + self.config.vision_config.crop_size + // self.config.vision_config.enc_patch_size + ) + enc_dim = self.config.vision_config.enc_dim + global_features = features[0] + + if features.shape[0] > 1: + if image_input.tiling is None: + raise ValueError( + "Missing tiling metadata for multi-crop Moondream image." + ) + local = features[1:].contiguous().view(-1, grid_size, grid_size, enc_dim) + reconstructed = reconstruct_from_crops( + local, + image_input.tiling, + overlap_margin=self.config.vision_config.overlap_margin, + patch_size=1, + ) + else: + reconstructed = global_features.view(grid_size, grid_size, enc_dim) + + recon = reconstructed.permute(2, 0, 1).contiguous() + # Mirror HF reference behavior: reconstructed local features are pooled + # to enc_n_layers x enc_n_layers. For moondream3-preview this is 27x27. + pooled_size = self.config.vision_config.enc_n_layers + if pooled_size != grid_size: + logger.warning_once( + "Moondream3 pooled_size (%d) differs from crop grid (%d). " + "Using enc_n_layers to match HF reference behavior.", + pooled_size, + grid_size, + ) + recon = F.adaptive_avg_pool2d(recon, output_size=(pooled_size, pooled_size)) + recon = recon.permute(1, 2, 0).contiguous().view(-1, enc_dim) + + combined = torch.cat([global_features, recon], dim=-1).unsqueeze(0) + projected = self.vision_proj(combined).squeeze(0) + + # Note: Vision embeddings are already synchronized across TP ranks + # because the vision projection uses RowParallelLinear which performs + # all-reduce internally, ensuring identical outputs on all ranks. + + return projected + + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: + """Generate the HF image prefix: BOS embedding + 729 image embeddings.""" + image_inputs = self._parse_image_inputs(**kwargs) + if not image_inputs: + return [] + + device = self.vision.patch_emb.weight.device + bos_ids = torch.tensor([self.config.bos_token_id], device=device) + bos_embedding = self.text.embed_input_ids(bos_ids) + + embeddings: list[torch.Tensor] = [] + for image_input in image_inputs: + image_embeddings = self._encode_image_input(image_input) + embeddings.append( + torch.cat([bos_embedding.to(image_embeddings.dtype), image_embeddings]) + ) + return embeddings + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor | IntermediateTensors: + hidden_states = self.text( + input_ids=input_ids, + positions=positions, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + ) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + logits = self.logits_processor(self.lm_head, hidden_states) + if logits is not None: + logits[:, self._answer_id] = float("-inf") + return logits + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + """Load weights with remapping from HuggingFace format.""" + + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + + # Get expert intermediate size for fc1 splitting + + for name, loaded_weight in weights: + # Map from HF naming to vLLM naming + # model.vision.* -> vision.* + # model.text.* -> text.* + if name.startswith("model."): + name = name[6:] # Remove "model." prefix + + # Specific name mappings + # Vision projection: vision.proj_mlp.fc1 -> vision_proj.fc1 + name = name.replace("vision.proj_mlp.", "vision_proj.") + + # Text embedding: text.wte (no suffix) -> text.wte.weight + if name == "text.wte": + name = "text.wte.weight" + + # LM head: text.lm_head -> lm_head + name = name.replace("text.lm_head.", "lm_head.") + + # Attention mapping + name = name.replace(".attn.qkv.", ".attn.qkv_proj.") + name = name.replace(".attn.proj.", ".attn.out_proj.") + + # Tau attention scaling weights + # HF format: .attn.tau.alpha -> .attn.tau_alpha + name = name.replace(".attn.tau.alpha", ".attn.tau_alpha") + name = name.replace(".attn.tau.wq", ".attn.tau_wq") + name = name.replace(".attn.tau.wv", ".attn.tau_wv") + + # MoE router mapping: mlp.router -> mlp.gate + name = name.replace(".mlp.router.", ".mlp.gate.") + + # Handle MoE expert weights for layers 4+ with expert parallelism + # fc1.weight: [n_experts, expert_inner_dim * 2, hidden_size] (gate+up) + # fc2.weight: [n_experts, hidden_size, expert_inner_dim] (down) + # Each GPU stores n_experts/tp_size experts + # Note: Only 3D weights are MoE, 2D weights are standard MLP + if ".mlp.fc1.weight" in name and loaded_weight.dim() == 3: + from vllm.distributed import get_tensor_model_parallel_rank + + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + num_experts = loaded_weight.shape[0] + experts_per_rank = num_experts // tp_size + expert_start = tp_rank * experts_per_rank + expert_end = expert_start + experts_per_rank + # Shard by expert dimension + loaded_weight = loaded_weight[expert_start:expert_end].contiguous() + # Map to our custom MoE format: mlp.fc1_weight + name = name.replace(".mlp.fc1.weight", ".mlp.fc1_weight") + + if ".mlp.fc2.weight" in name and loaded_weight.dim() == 3: + from vllm.distributed import get_tensor_model_parallel_rank + + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + num_experts = loaded_weight.shape[0] + experts_per_rank = num_experts // tp_size + expert_start = tp_rank * experts_per_rank + expert_end = expert_start + experts_per_rank + # Shard by expert dimension + loaded_weight = loaded_weight[expert_start:expert_end].contiguous() + # Map to our custom MoE format: mlp.fc2_weight + name = name.replace(".mlp.fc2.weight", ".mlp.fc2_weight") + + # Handle tau weights with tensor parallelism + # tau_alpha: [num_heads] -> [num_heads/tp] + # tau_wq: [num_heads, qkv_dim] -> [num_heads/tp, qkv_dim/tp] + # tau_wv: [num_heads, qkv_dim] -> [num_heads/tp, qkv_dim/tp] + if ".tau_alpha" in name: + from vllm.distributed import get_tensor_model_parallel_rank + + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + num_heads = loaded_weight.shape[0] + heads_per_partition = num_heads // tp_size + start = tp_rank * heads_per_partition + end = start + heads_per_partition + loaded_weight = loaded_weight[start:end].contiguous() + + if ".tau_wq" in name or ".tau_wv" in name: + from vllm.distributed import get_tensor_model_parallel_rank + + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + num_heads, qkv_dim = loaded_weight.shape + heads_per_partition = num_heads // tp_size + # Only shard by head dimension, keep full qkv_dim for all-gather + head_start = tp_rank * heads_per_partition + head_end = head_start + heads_per_partition + loaded_weight = loaded_weight[head_start:head_end, :].contiguous() + + if name in params_dict: + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + + return loaded_params diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 9a9884807e..3cce0733bf 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -484,6 +484,8 @@ _MULTIMODAL_MODELS = { ), "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"), "Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"), + "Moondream3ForCausalLM": ("moondream3", "Moondream3ForCausalLM"), + "HfMoondream": ("moondream3", "Moondream3ForCausalLM"), "NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"), "NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"), "NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"), diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 47b74093b0..2f00178ba6 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -114,6 +114,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( mlp_speculator="MLPSpeculatorConfig", medusa="MedusaConfig", midashenglm="MiDashengLMConfig", + moondream3="Moondream3Config", eagle="EAGLEConfig", speculators="SpeculatorsConfig", nemotron="NemotronConfig", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 8c4d01a428..44abe32c91 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -50,6 +50,9 @@ _CLASS_TO_MODULE: dict[str, str] = { "MedusaConfig": "vllm.transformers_utils.configs.medusa", "MiDashengLMConfig": "vllm.transformers_utils.configs.midashenglm", "MLPSpeculatorConfig": "vllm.transformers_utils.configs.mlp_speculator", + "Moondream3Config": "vllm.transformers_utils.configs.moondream3", + "Moondream3TextConfig": "vllm.transformers_utils.configs.moondream3", + "Moondream3VisionConfig": "vllm.transformers_utils.configs.moondream3", "MoonViTConfig": "vllm.transformers_utils.configs.moonvit", "KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear", "KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl", @@ -111,6 +114,9 @@ __all__ = [ "MedusaConfig", "MiDashengLMConfig", "MLPSpeculatorConfig", + "Moondream3Config", + "Moondream3TextConfig", + "Moondream3VisionConfig", "MoonViTConfig", "KimiLinearConfig", "KimiVLConfig", diff --git a/vllm/transformers_utils/configs/moondream3.py b/vllm/transformers_utils/configs/moondream3.py new file mode 100644 index 0000000000..307bb29772 --- /dev/null +++ b/vllm/transformers_utils/configs/moondream3.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Configuration for Moondream3 model.""" + +from transformers import PretrainedConfig + + +class Moondream3VisionConfig(PretrainedConfig): + """Vision encoder configuration for Moondream3.""" + + model_type = "moondream3_vision" + + def __init__( + self, + enc_dim: int = 1152, + enc_patch_size: int = 14, + enc_n_layers: int = 27, + enc_ff_dim: int = 4304, + enc_n_heads: int = 16, + proj_inner_dim: int = 8192, + crop_size: int = 378, + max_crops: int = 12, + overlap_margin: int = 4, + **kwargs, + ): + super().__init__(**kwargs) + self.enc_dim = enc_dim + self.enc_patch_size = enc_patch_size + self.enc_n_layers = enc_n_layers + self.enc_ff_dim = enc_ff_dim + self.enc_n_heads = enc_n_heads + self.proj_inner_dim = proj_inner_dim + self.crop_size = crop_size + self.max_crops = max_crops + self.overlap_margin = overlap_margin + + # Standard HuggingFace attributes for vision config + self.hidden_size = enc_dim + self.num_attention_heads = enc_n_heads + self.num_hidden_layers = enc_n_layers + self.intermediate_size = enc_ff_dim + self.patch_size = enc_patch_size + self.image_size = crop_size + + +class Moondream3TextConfig(PretrainedConfig): + """Text decoder configuration for Moondream3.""" + + model_type = "moondream3_text" + + def __init__( + self, + dim: int = 2048, + ff_dim: int = 8192, + n_layers: int = 24, + vocab_size: int = 51200, + max_context: int = 4096, + n_heads: int = 32, + n_kv_heads: int = 32, + prefix_attn: int = 730, + rope_theta: float = 1500000.0, + moe: dict | None = None, + **kwargs, + ): + super().__init__(**kwargs) + + # Store original moondream3 config names + self.dim = dim + self.ff_dim = ff_dim + self.n_layers = n_layers + self.n_heads = n_heads + self.n_kv_heads = n_kv_heads + self.prefix_attn = prefix_attn + self.max_context = max_context + self.rope_theta = rope_theta + + # MoE config + moe = moe or {} + self.moe_start_layer = moe.get("start_layer", 4) + self.moe_num_experts = moe.get("n_experts", 64) + self.moe_experts_per_token = moe.get("n_experts_per_tok", 8) + self.moe_expert_inner_dim = moe.get("expert_inner_dim", 1024) + + # Standard HuggingFace attributes (required by vLLM) + self.hidden_size = dim + self.num_attention_heads = n_heads + self.num_key_value_heads = n_kv_heads + self.num_hidden_layers = n_layers + self.intermediate_size = ff_dim + self.vocab_size = vocab_size + self.max_position_embeddings = max_context + + # Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS. + # Token 3 (<|md_reserved_2|>) is an answer delimiter that the model + # implementation suppresses during generation. + self.bos_token_id = 0 + self.eos_token_id = 0 + + # MoE standard attributes + self.num_local_experts = self.moe_num_experts + self.num_experts_per_tok = self.moe_experts_per_token + + +class Moondream3Config(PretrainedConfig): + """Combined configuration for Moondream3 multimodal model.""" + + model_type = "moondream3" + is_composition = True + + def __init__( + self, + config: dict | None = None, + **kwargs, + ): + config = config or {} + + # Parse text config + text_config = config.get("text", {}) + self.text_config: Moondream3TextConfig = Moondream3TextConfig(**text_config) + + # Parse vision config + vision_config = config.get("vision", {}) + self.vision_config = Moondream3VisionConfig(**vision_config) + + # Store the original config dict for model access + self.config = config + tokenizer_config = config.get("tokenizer", {}) + self.answer_token_id = tokenizer_config.get("answer_id", 3) + + super().__init__(**kwargs) + + # Expose key attributes at top level for vLLM compatibility + self.hidden_size = self.text_config.hidden_size + self.num_attention_heads = self.text_config.num_attention_heads + self.num_key_value_heads = self.text_config.num_key_value_heads + self.num_hidden_layers = self.text_config.num_hidden_layers + self.vocab_size = self.text_config.vocab_size + self.intermediate_size = self.text_config.intermediate_size + + # Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS. + # Token 3 (<|md_reserved_2|>) is an answer delimiter that the model + # implementation suppresses during generation. + self.bos_token_id = 0 + self.eos_token_id = 0 + + def get_text_config(self, decoder: bool = False) -> "Moondream3TextConfig": + """Return the text config for vLLM's text_config detection. + + Args: + decoder: Ignored. Only used for encoder-decoder models. + """ + return self.text_config diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index b0f290649d..79f28d815e 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -267,6 +267,7 @@ class ModelArchConfigConvertorBase: "bagel", "gemma3", "molmo2", + "moondream3", "paligemma", "umm", ) diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index 546a5c4532..4f44591b16 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -23,6 +23,7 @@ __all__ = [ "H2OVLProcessor", "HunYuanVLProcessor", "HunYuanVLImageProcessor", + "Moondream3Processor", "InternVLProcessor", "IsaacProcessor", "KimiAudioProcessor", @@ -61,6 +62,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "MiMoOmniProcessor": "vllm.transformers_utils.processors.mimo_v2_omni", "MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral", "MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral", + "Moondream3Processor": "vllm.transformers_utils.processors.moondream3", "NanoNemotronVLProcessor": "vllm.transformers_utils.processors.nano_nemotron_vl", "NemotronVLProcessor": "vllm.transformers_utils.processors.nemotron_vl", "LlamaNemotronVLEmbedProcessor": "vllm.transformers_utils.processors.nemotron_vl", diff --git a/vllm/transformers_utils/processors/moondream3.py b/vllm/transformers_utils/processors/moondream3.py new file mode 100644 index 0000000000..289c40dd17 --- /dev/null +++ b/vllm/transformers_utils/processors/moondream3.py @@ -0,0 +1,541 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Custom processor for Moondream3 model.""" + +import math + +import numpy as np +import torch +from PIL import Image +from transformers import AutoProcessor, BatchFeature +from transformers.image_utils import ImageInput +from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack +from transformers.tokenization_utils_base import ( + PreTokenizedInput, + PreTrainedTokenizerBase, + TextInput, +) + +from vllm.multimodal.image import convert_image_mode + +__all__ = ["Moondream3Processor"] + + +class Moondream3ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] + _defaults = { + "text_kwargs": { + "padding": False, + }, + "images_kwargs": { + "max_crops": 12, + "overlap_margin": 4, + "crop_size": 378, + "patch_size": 14, + "convert_to_rgb": True, + "return_tensors": "pt", + }, + } + + +def select_tiling( + height: int, width: int, crop_size: int, max_crops: int +) -> tuple[int, int]: + """Determine the optimal number of tiles to cover an image.""" + if height <= crop_size or width <= crop_size: + return (1, 1) + + min_h = math.ceil(height / crop_size) + min_w = math.ceil(width / crop_size) + + if min_h * min_w > max_crops: + ratio = math.sqrt(max_crops / (min_h * min_w)) + return (max(1, math.floor(min_h * ratio)), max(1, math.floor(min_w * ratio))) + + h_tiles = math.floor(math.sqrt(max_crops * height / width)) + w_tiles = math.floor(math.sqrt(max_crops * width / height)) + + h_tiles = max(h_tiles, min_h) + w_tiles = max(w_tiles, min_w) + + if h_tiles * w_tiles > max_crops: + if w_tiles > h_tiles: + w_tiles = math.floor(max_crops / h_tiles) + else: + h_tiles = math.floor(max_crops / w_tiles) + + return (max(1, h_tiles), max(1, w_tiles)) + + +class Moondream3Processor(ProcessorMixin): + """ + Constructs a Moondream3 processor which handles image preprocessing + and tokenization for the Moondream3 multimodal model. + + Args: + tokenizer: The tokenizer to use for text processing. + chat_template: Optional chat template string. + crop_size: Size of each image crop. + max_crops: Maximum number of crops per image. + overlap_margin: Margin for overlapping crops in patches. + patch_size: Size of each patch. + """ + + attributes = ["tokenizer"] + valid_kwargs = [ + "chat_template", + "crop_size", + "max_crops", + "overlap_margin", + "patch_size", + ] + + tokenizer_class = "AutoTokenizer" + # Use separate tokenizer repo + _tokenizer_repo = "moondream/starmie-v1" + + # Default chat template for Moondream3 + # Moondream uses special tokens for prompting: + # - Token 0 (<|endoftext|>): BOS token (ALWAYS present at position 0) + # - Token 1 (<|md_reserved_0|>): Start of instruction + # - Token 2 (<|md_reserved_1|>): Separator before question + # - Token 3 (<|md_reserved_2|>): End of question / start of answer + # + # Task routing based on text prefix: + # "caption [short|normal|long]" → describe<|md_reserved_1|>{length} + # "describe [short|normal|long]" → describe<|md_reserved_1|>{length} + # otherwise → query<|md_reserved_1|> + # + # Format with image: + # <|endoftext|><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|> + # Format without image: + # <|endoftext|><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|> + _default_chat_template = ( + "{% for message in messages %}" + "{% if message['role'] == 'user' %}" + "{% if message['content'] is string %}" + # Simple string content (with image assumed) - route by prefix + "<|endoftext|><|md_reserved_0|>" + "{% if message['content'] == 'caption' %}" + "describe<|md_reserved_1|>normal<|md_reserved_2|>" + "{% elif message['content'].startswith('caption ') %}" + "describe<|md_reserved_1|>{{ message['content'][8:] }}<|md_reserved_2|>" + "{% elif message['content'] == 'describe' %}" + "describe<|md_reserved_1|>normal<|md_reserved_2|>" + "{% elif message['content'].startswith('describe ') %}" + "describe<|md_reserved_1|>{{ message['content'][9:] }}<|md_reserved_2|>" + "{% else %}" + "query<|md_reserved_1|>{{ message['content'] }}<|md_reserved_2|>" + "{% endif %}" + "{% else %}" + # List content - build Moondream's image prefix independently of + # OpenAI-style content part order, then render the text task. + "<|endoftext|>" + "{% for content in message['content'] %}" + "{% if content['type'] in ['image', 'image_url', 'input_image', 'image_pil'] %}" # noqa: E501 + "" + "{% endif %}" + "{% endfor %}" + "{% for content in message['content'] %}" + "{% if content['type'] == 'text' %}" + "<|md_reserved_0|>" + "{% if content['text'] == 'caption' %}" + "describe<|md_reserved_1|>normal<|md_reserved_2|>" + "{% elif content['text'].startswith('caption ') %}" + "describe<|md_reserved_1|>{{ content['text'][8:] }}<|md_reserved_2|>" + "{% elif content['text'] == 'describe' %}" + "describe<|md_reserved_1|>normal<|md_reserved_2|>" + "{% elif content['text'].startswith('describe ') %}" + "describe<|md_reserved_1|>{{ content['text'][9:] }}<|md_reserved_2|>" + "{% else %}" + "query<|md_reserved_1|>{{ content['text'] }}<|md_reserved_2|>" + "{% endif %}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "{% elif message['role'] == 'assistant' %}" + "{{ message['content'] }}" + "{% endif %}" + "{% endfor %}" + ) + + def __init__( + self, + tokenizer: PreTrainedTokenizerBase | None = None, + chat_template: str | None = None, + crop_size: int = 378, + max_crops: int = 12, + overlap_margin: int = 4, + patch_size: int = 14, + **kwargs, + ): + self.image_token = "" + self.crop_size = crop_size + self.max_crops = max_crops + self.overlap_margin = overlap_margin + self.patch_size = patch_size + + # Number of patches per crop (27x27 = 729 for 378/14) + self.patches_per_crop = (crop_size // patch_size) ** 2 + + # Use default chat template if none provided + if chat_template is None: + chat_template = self._default_chat_template + + super().__init__(tokenizer, chat_template=chat_template) + + @classmethod + def from_pretrained( + cls, + pretrained_model_name_or_path, + **kwargs, + ): + """ + Load the processor, using a separate tokenizer repo. + + The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1' + instead of having tokenizer files in the model repo. + """ + from transformers import AutoTokenizer, PreTrainedTokenizerFast + from transformers.utils import cached_file + + tokenizer = kwargs.pop("tokenizer", None) + + tokenizer_kwargs = { + "trust_remote_code": kwargs.get("trust_remote_code", False), + } + for key in ( + "cache_dir", + "force_download", + "local_files_only", + "revision", + "subfolder", + "token", + "use_fast", + ): + if key in kwargs: + tokenizer_kwargs[key] = kwargs[key] + + cached_file_kwargs = { + key: tokenizer_kwargs[key] + for key in ( + "cache_dir", + "force_download", + "local_files_only", + "revision", + "subfolder", + "token", + ) + if key in tokenizer_kwargs + } + + def load_tokenizer(repo_or_path): + try: + return AutoTokenizer.from_pretrained(repo_or_path, **tokenizer_kwargs) + except Exception: + tokenizer_file = cached_file( + repo_or_path, + "tokenizer.json", + **cached_file_kwargs, + ) + return PreTrainedTokenizerFast( + tokenizer_file=tokenizer_file, + clean_up_tokenization_spaces=False, + ) + + if isinstance(tokenizer, str): + tokenizer = load_tokenizer(tokenizer) + + if tokenizer is None: + # Prefer model-local tokenizer files first. If unavailable, fall + # back to moondream's dedicated tokenizer repository. + try: + tokenizer = load_tokenizer(pretrained_model_name_or_path) + except Exception: + tokenizer = load_tokenizer(cls._tokenizer_repo) + + # Configure special tokens for Moondream3 + # BOS and EOS are both token 0 (<|endoftext|>), matching the native + # config (TokenizerConfig.bos_id=0, eos_id=0). This is standard for + # GPT-2 style models where <|endoftext|> signals both start and end. + # Token 1 (<|md_reserved_0|>) is a template delimiter, NOT the EOS. + tokenizer.bos_token = "<|endoftext|>" + tokenizer.bos_token_id = 0 + tokenizer.eos_token = "<|endoftext|>" + tokenizer.eos_token_id = 0 + + # Extract processor-specific kwargs + crop_size = kwargs.pop("crop_size", 378) + max_crops = kwargs.pop("max_crops", 12) + overlap_margin = kwargs.pop("overlap_margin", 4) + patch_size = kwargs.pop("patch_size", 14) + chat_template = kwargs.pop("chat_template", None) + + # Set default chat template on tokenizer if not already set + if chat_template is None: + chat_template = cls._default_chat_template + if tokenizer.chat_template is None: + tokenizer.chat_template = chat_template + + return cls( + tokenizer=tokenizer, + chat_template=chat_template, + crop_size=crop_size, + max_crops=max_crops, + overlap_margin=overlap_margin, + patch_size=patch_size, + ) + + def __call__( + self, + images: ImageInput = None, + text: TextInput + | PreTokenizedInput + | list[TextInput] + | list[PreTokenizedInput] = None, + **kwargs: Unpack[Moondream3ProcessorKwargs], + ) -> BatchFeature: + """ + Process images and text for Moondream3 model. + + Args: + images: Input images (PIL Image, numpy array, or list thereof). + text: Input text or list of texts. + **kwargs: Additional processing arguments. + + Returns: + BatchFeature with processed inputs. + """ + output_kwargs = self._merge_kwargs( + Moondream3ProcessorKwargs, + tokenizer_init_kwargs=self.tokenizer.init_kwargs, + **kwargs, + ) + + # Process images + image_features = {} + if images is not None: + processed_images = [] + tilings = [] + + images_list = images if isinstance(images, list) else [images] + for image in images_list: + pixel_values, tiling = self.preprocess_image( + image, **output_kwargs["images_kwargs"] + ) + processed_images.append(pixel_values) + tilings.append(tiling) + + if processed_images: + image_features["pixel_values"] = processed_images + image_features["tilings"] = tilings + + # Process text + if text is not None: + if not isinstance(text, list): + text = [text] + + # Get text kwargs, remove keys we set ourselves + text_kwargs = output_kwargs.get("text_kwargs", {}).copy() + text_kwargs.pop("return_tensors", None) + text_kwargs.pop("add_special_tokens", None) + + # Tokenize text + tokenized = self.tokenizer( + text, + add_special_tokens=True, + return_tensors="pt", + **text_kwargs, + ) + + output = BatchFeature(data=dict(tokenized)) + + # Add image features + if image_features: + output["pixel_values"] = image_features["pixel_values"] + output["tilings"] = image_features["tilings"] + + return output + + # If only images were provided + return BatchFeature(data=image_features) + + @staticmethod + def _image_array_to_uint8(array: np.ndarray) -> np.ndarray: + if array.dtype == np.uint8: + return np.ascontiguousarray(array) + + if array.dtype == np.bool_: + return np.ascontiguousarray(array.astype(np.uint8) * 255) + + if np.issubdtype(array.dtype, np.floating): + array = np.nan_to_num(array, nan=0.0, posinf=255.0, neginf=0.0) + if array.size > 0 and array.max() <= 1.0: + array = array * 255.0 + array = np.rint(array) + + return np.ascontiguousarray(np.clip(array, 0, 255).astype(np.uint8)) + + @staticmethod + def _to_pil_image(image: ImageInput) -> Image.Image: + if isinstance(image, Image.Image): + return image + + if isinstance(image, torch.Tensor): + tensor = image.detach().cpu() + if tensor.dtype == torch.bfloat16: + tensor = tensor.to(torch.float32) + image_array = tensor.numpy() + elif isinstance(image, np.ndarray): + image_array = image + else: + raise TypeError( + "Moondream3 images must be PIL images, numpy arrays, " + f"or torch tensors, got {type(image)!r}." + ) + + if image_array.ndim == 2: + image_array = Moondream3Processor._image_array_to_uint8(image_array) + return Image.fromarray(image_array) + + if image_array.ndim != 3: + raise ValueError( + "Moondream3 image arrays must have 2 or 3 dimensions, " + f"got shape {image_array.shape}." + ) + + channel_dims = (1, 3, 4) + if image_array.shape[-1] not in channel_dims: + if image_array.shape[0] not in channel_dims: + raise ValueError( + "Moondream3 image arrays must be HWC or CHW with 1, 3, " + f"or 4 channels, got shape {image_array.shape}." + ) + image_array = np.transpose(image_array, (1, 2, 0)) + + image_array = Moondream3Processor._image_array_to_uint8(image_array) + if image_array.shape[-1] == 1: + image_array = image_array[..., 0] + + return Image.fromarray(image_array) + + def preprocess_image( + self, + image: ImageInput, + max_crops: int = 12, + overlap_margin: int = 4, + crop_size: int = 378, + patch_size: int = 14, + convert_to_rgb: bool = True, + return_tensors: str = "pt", + ) -> tuple[torch.Tensor, tuple[int, int]]: + """ + Preprocess an image using overlap-and-resize cropping strategy. + + Args: + image: Input PIL image, numpy array, or torch tensor. + max_crops: Maximum number of crops. + overlap_margin: Margin for overlapping in patches. + crop_size: Size of each crop. + patch_size: Size of each patch. + convert_to_rgb: Whether to convert to RGB. + return_tensors: Return type ("pt" for PyTorch). + + Returns: + Tuple of (pixel_values tensor, tiling tuple). + """ + image = self._to_pil_image(image) + if convert_to_rgb: + image = convert_image_mode(image, "RGB") + + # Convert to numpy array + image_array = np.array(image) + original_h, original_w = image_array.shape[:2] + + margin_pixels = patch_size * overlap_margin + total_margin_pixels = margin_pixels * 2 + + crop_patches = crop_size // patch_size + crop_window_patches = crop_patches - (2 * overlap_margin) + crop_window_size = crop_window_patches * patch_size + + tiling = select_tiling( + original_h - total_margin_pixels, + original_w - total_margin_pixels, + crop_window_size, + max_crops, + ) + + n_crops = tiling[0] * tiling[1] + 1 + crops = np.zeros((n_crops, crop_size, crop_size, 3), dtype=np.uint8) + + target_size = ( + tiling[0] * crop_window_size + total_margin_pixels, + tiling[1] * crop_window_size + total_margin_pixels, + ) + + # Resize image + pil_img = Image.fromarray(image_array) + resized = pil_img.resize( + (int(target_size[1]), int(target_size[0])), + resample=Image.Resampling.LANCZOS, + ) + resized_array = np.asarray(resized) + + # Create global crop + global_pil = pil_img.resize( + (crop_size, crop_size), resample=Image.Resampling.LANCZOS + ) + crops[0] = np.asarray(global_pil) + + # Create local crops + for i in range(tiling[0]): + for j in range(tiling[1]): + y0 = i * crop_window_size + x0 = j * crop_window_size + y_end = min(y0 + crop_size, resized_array.shape[0]) + x_end = min(x0 + crop_size, resized_array.shape[1]) + + crop_region = resized_array[y0:y_end, x0:x_end] + crop_idx = 1 + i * tiling[1] + j + h_slice = slice(None, crop_region.shape[0]) + w_slice = slice(None, crop_region.shape[1]) + crops[crop_idx, h_slice, w_slice] = crop_region + + # Convert to tensor: (n_crops, H, W, C) -> (n_crops, C, H, W) + pixel_values = np.transpose(crops, (0, 3, 1, 2)) + + if return_tensors == "pt": + # Match HF reference preprocessing exactly: convert uint8 crops to + # bfloat16 before in-place normalization. + pixel_values = ( + torch.from_numpy(pixel_values) + .to(dtype=torch.bfloat16) + .div_(255.0) + .sub_(0.5) + .div_(0.5) + ) + else: + pixel_values = pixel_values.astype(np.float32) / 255.0 + pixel_values = (pixel_values - 0.5) / 0.5 + + return pixel_values, tiling + + def get_num_image_tokens(self) -> int: + """Return the number of image tokens (729 = 27x27 patches).""" + return self.patches_per_crop + + def batch_decode(self, *args, **kwargs): + """Forward to tokenizer's batch_decode.""" + return self.tokenizer.batch_decode(*args, **kwargs) + + def decode(self, *args, **kwargs): + """Forward to tokenizer's decode.""" + return self.tokenizer.decode(*args, **kwargs) + + @property + def model_input_names(self): + tokenizer_input_names = self.tokenizer.model_input_names + return tokenizer_input_names + ["pixel_values", "tilings"] + + +AutoProcessor.register("Moondream3Processor", Moondream3Processor)