[Model] Add Moondream3 model support(only query and caption skills) (#32325)

Signed-off-by: Dong Wang <dongw2019@gmail.com>
This commit is contained in:
Dong W
2026-05-01 10:06:48 +08:00
committed by GitHub
parent 14043dfecd
commit 7198940b39
19 changed files with 3238 additions and 9 deletions
+61
View File
@@ -215,6 +215,67 @@ When loading RGBA images (images with transparency), vLLM converts them to RGB f
- This setting only affects RGBA images with transparency; RGB images are unchanged
- If not specified, the default white background `(255, 255, 255)` is used for backward compatibility
#### Moondream3 Prompt Recipes { #moondream3-prompt-recipes }
`Moondream3ForCausalLM` supports two task-specific prompt formats:
- `query`: ask a question about the image.
- `caption`: generate a caption for the image.
```python
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
llm = LLM(
model="moondream/moondream3-preview",
tokenizer="moondream/starmie-v1",
trust_remote_code=True,
max_model_len=2048,
limit_mm_per_prompt={"image": 1},
)
image = ImageAsset("stop_sign").pil_image
def make_query_prompt(question: str) -> str:
return (
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
f"{question}<|md_reserved_2|>"
)
def make_caption_prompt(length: str = "normal") -> str:
return (
"<|endoftext|><image><|md_reserved_0|>"
f"describe<|md_reserved_1|>{length}<|md_reserved_2|>"
)
query_out = llm.generate(
{
"prompt": make_query_prompt("What is shown in this image?"),
"multi_modal_data": {"image": image},
},
SamplingParams(max_tokens=64, temperature=0),
)[0].outputs[0].text
caption_out = llm.generate(
{
"prompt": make_caption_prompt(),
"multi_modal_data": {"image": image},
},
SamplingParams(max_tokens=100, temperature=0),
)[0].outputs[0].text
print("query:", query_out)
print("caption:", caption_out)
```
!!! note
The native Moondream3 model also has `detect` and `point` skills. Those
require custom coordinate decoding and are not exposed by this vLLM
implementation.
### Video Inputs
You can pass a list of NumPy arrays directly to the `'video'` field of the multi-modal dictionary
+7
View File
@@ -599,6 +599,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I<sup>+</sup> | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ |
| `MolmoForCausalLM` | Molmo | T + I<sup>+</sup> | `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc. | ✅︎ | ✅︎ |
| `Molmo2ForConditionalGeneration` | Molmo2 | T + I<sup>+</sup> / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B` | ✅︎ | ✅︎ |
| `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |
| `MusicFlamingoForConditionalGeneration` | MusicFlamingo | T + A | `nvidia/music-flamingo-2601-hf`, `nvidia/music-flamingo-think-2601-hf` | ✅︎ | ✅︎ |
| `NVLM_D_Model` | NVLM-D 1.0 | T + I<sup>+</sup> | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
| `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + I<sup>E+</sup> | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |
@@ -661,6 +662,12 @@ Some models are supported only via the [Transformers modeling backend](#transfor
!!! note
For `InternVLChatModel`, only InternVL2.5 with Qwen2.5 text backbone (`OpenGVLab/InternVL2.5-1B` etc.), InternVL3 and InternVL3.5 have video inputs support currently.
!!! note
`Moondream3ForCausalLM` uses task-specific prompt templates for `query`
and `caption`. The native `detect` and `point` skills require custom
coordinate decoding and are not exposed by this vLLM implementation.
See [Moondream3 prompt recipes](../features/multimodal_inputs.md#moondream3-prompt-recipes).
!!! note
To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
+31 -8
View File
@@ -371,6 +371,8 @@ class HfRunner:
is_cross_encoder: bool = False,
skip_tokenizer_init: bool = False,
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
tokenizer_name: str | None = None,
processor: Any | None = None,
# Set this to avoid hanging issue
default_torch_num_threads: int | None = None,
) -> None:
@@ -391,6 +393,8 @@ class HfRunner:
is_cross_encoder=is_cross_encoder,
skip_tokenizer_init=skip_tokenizer_init,
auto_cls=auto_cls,
tokenizer_name=tokenizer_name,
processor=processor,
)
def _init(
@@ -405,6 +409,8 @@ class HfRunner:
is_cross_encoder: bool = False,
skip_tokenizer_init: bool = False,
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
tokenizer_name: str | None = None,
processor: Any | None = None,
) -> None:
model_name = maybe_model_redirect(model_name)
self.model_name = model_name
@@ -484,20 +490,27 @@ class HfRunner:
if not skip_tokenizer_init:
self.tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast" = (
AutoTokenizer.from_pretrained(
model_name,
tokenizer_name or model_name,
trust_remote_code=trust_remote_code,
)
)
# don't put this import at the top level
# it will call torch.accelerator.device_count()
from transformers import AutoProcessor
if processor is not None:
self.processor = processor
else:
# don't put this import at the top level
# it will call torch.accelerator.device_count()
from transformers import AutoProcessor
self.processor = AutoProcessor.from_pretrained(
model_name,
trust_remote_code=trust_remote_code,
)
self.processor = AutoProcessor.from_pretrained(
model_name,
trust_remote_code=trust_remote_code,
)
if skip_tokenizer_init:
if self.processor is None:
raise ValueError(
"skip_tokenizer_init=True requires processor initialization."
)
self.tokenizer = self.processor.tokenizer
def get_inputs(
@@ -520,6 +533,12 @@ class HfRunner:
all_inputs: list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]] = []
for i, prompt in enumerate(prompts):
if isinstance(prompt, str):
if self.processor is None:
raise RuntimeError(
"HfRunner.processor is not initialized. "
"Pass processor=... to HfRunner or set "
"hf_model.processor before generation."
)
# Create a copy to avoid modifying the original dict
processor_kwargs = (
tokenization_kwargs.copy()
@@ -617,6 +636,10 @@ class HfRunner:
use_cache=True,
**kwargs,
)
if self.processor is None:
raise RuntimeError(
"HfRunner.processor is not initialized; cannot decode output."
)
output_str = self.processor.batch_decode(
output_ids,
skip_special_tokens=True,
@@ -805,6 +805,32 @@ VLM_TEST_SETTINGS = {
max_num_seqs=2,
patch_hf_runner=model_utils.molmo_patch_hf_runner,
),
"moondream3": VLMTestInfo(
models=["moondream/moondream3-preview"],
test_type=VLMTestType.IMAGE,
prompt_formatter=identity,
img_idx_to_prompt=lambda idx: "<|endoftext|><image>",
# Common-image coverage here targets query/caption. The native
# detect/point skills are not exposed by vLLM.
single_image_prompts=IMAGE_ASSETS.prompts(
{
"stop_sign": "<vlm_image><|md_reserved_0|>query<|md_reserved_1|>What is this sign?<|md_reserved_2|>", # noqa: E501
"cherry_blossom": (
"<vlm_image><|md_reserved_0|>query<|md_reserved_1|>What season is shown?<|md_reserved_2|>" # noqa: E501
),
}
),
max_model_len=4096,
max_num_seqs=2,
dtype="bfloat16",
hf_processor=model_utils.moondream3_processor,
patch_hf_runner=model_utils.moondream3_patch_hf_runner,
# Single size factor to avoid GPU OOM when running multiple test
# cases sequentially (9B MoE model uses ~18 GiB per instance).
image_size_factors=[(1.0,)],
# Moondream3 is 9B params with MoE, needs significant GPU memory
marks=[large_gpu_mark(min_gb=48)],
),
"ovis1_6-gemma2": VLMTestInfo(
models=["AIDC-AI/Ovis1.6-Gemma2-9B"],
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
@@ -0,0 +1,176 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Generation tests for Moondream3 query and caption support."""
import pytest
from tests.models.registry import HF_EXAMPLE_MODELS
from vllm.platforms import current_platform
from ....conftest import IMAGE_ASSETS, ImageTestAssets
from ....utils import large_gpu_mark, multi_gpu_test
MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
MOONDREAM3_TOKENIZER = "moondream/starmie-v1"
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What color is the stop sign?<|md_reserved_2|>", # noqa: E501
"cherry_blossom": "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What color are the flowers?<|md_reserved_2|>", # noqa: E501
}
)
def make_query_prompt(question: str) -> str:
"""Create a direct-answer query prompt for Moondream3."""
return (
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
f"{question}<|md_reserved_2|>"
)
def make_caption_prompt(length: str = "normal") -> str:
"""Create a caption prompt for Moondream3."""
return (
"<|endoftext|><image><|md_reserved_0|>"
f"describe<|md_reserved_1|>{length}<|md_reserved_2|>"
)
@multi_gpu_test(num_gpus=2)
@large_gpu_mark(min_gb=80)
def test_tensor_parallel(image_assets: ImageTestAssets):
import gc
from vllm import LLM, SamplingParams
from vllm.distributed.parallel_state import destroy_model_parallel
destroy_model_parallel()
gc.collect()
current_platform.empty_cache()
llm = LLM(
model=MOONDREAM3_MODEL_ID,
tokenizer=MOONDREAM3_TOKENIZER,
trust_remote_code=True,
dtype="bfloat16",
tensor_parallel_size=2,
max_model_len=1024,
enforce_eager=True,
limit_mm_per_prompt={"image": 1},
gpu_memory_utilization=0.45,
)
image = image_assets[0].pil_image
prompt = make_query_prompt("What color is the stop sign?")
try:
outputs = llm.generate(
{"prompt": prompt, "multi_modal_data": {"image": image}},
SamplingParams(max_tokens=20, temperature=0),
)
assert len(outputs) > 0
assert outputs[0].outputs[0].text is not None
finally:
del llm
gc.collect()
current_platform.empty_cache()
@pytest.fixture(scope="module")
def llm():
model_info = HF_EXAMPLE_MODELS.get_hf_info("Moondream3ForCausalLM")
model_info.check_transformers_version(on_fail="skip")
from vllm import LLM
try:
return LLM(
model=MOONDREAM3_MODEL_ID,
tokenizer=MOONDREAM3_TOKENIZER,
trust_remote_code=True,
dtype="bfloat16",
max_model_len=2048,
enforce_eager=True,
limit_mm_per_prompt={"image": 1},
gpu_memory_utilization=0.45,
)
except Exception as exc:
pytest.skip(f"Failed to load {MOONDREAM3_MODEL_ID}: {exc}")
@large_gpu_mark(min_gb=48)
def test_model_loading(llm):
assert llm is not None
@large_gpu_mark(min_gb=48)
def test_query_skill(llm, image_assets: ImageTestAssets):
from vllm import SamplingParams
image = image_assets[0].pil_image
prompt = make_query_prompt("What color is the stop sign?")
outputs = llm.generate(
{"prompt": prompt, "multi_modal_data": {"image": image}},
SamplingParams(max_tokens=50, temperature=0),
)
output_text = outputs[0].outputs[0].text
assert output_text is not None
assert len(output_text) > 0
@large_gpu_mark(min_gb=48)
def test_caption_skill(llm, image_assets: ImageTestAssets):
from vllm import SamplingParams
image = image_assets[1].pil_image
prompt = make_caption_prompt()
outputs = llm.generate(
{"prompt": prompt, "multi_modal_data": {"image": image}},
SamplingParams(max_tokens=100, temperature=0),
)
output_text = outputs[0].outputs[0].text
assert output_text is not None
assert len(output_text) > 0
@large_gpu_mark(min_gb=48)
def test_batched_inference(llm, image_assets: ImageTestAssets):
from vllm import SamplingParams
images = [asset.pil_image for asset in image_assets]
prompts = [
{"prompt": prompt, "multi_modal_data": {"image": img}}
for img, prompt in zip(images, HF_IMAGE_PROMPTS)
]
outputs = llm.generate(prompts, SamplingParams(max_tokens=50, temperature=0))
assert len(outputs) == len(images)
for output in outputs:
assert output.outputs[0].text is not None
assert len(output.outputs[0].text) > 0
@pytest.mark.parametrize("asset_name", ["stop_sign", "cherry_blossom"])
@large_gpu_mark(min_gb=48)
def test_image_assets(llm, image_assets: ImageTestAssets, asset_name: str):
from vllm import SamplingParams
asset_idx = 0 if asset_name == "stop_sign" else 1
image = image_assets[asset_idx].pil_image
prompt = HF_IMAGE_PROMPTS[asset_idx]
outputs = llm.generate(
{"prompt": prompt, "multi_modal_data": {"image": image}},
SamplingParams(max_tokens=50, temperature=0),
)
output_text = outputs[0].outputs[0].text
assert output_text is not None
assert len(output_text) > 0
@@ -38,6 +38,7 @@ def run_test(
limit_mm_per_prompt: dict[str, int],
vllm_runner_kwargs: dict[str, Any] | None,
hf_model_kwargs: dict[str, Any] | None,
hf_processor: Callable[[str], Any] | None,
patch_hf_runner: Callable[[HfRunner], HfRunner] | None,
runner: RunnerOption = "auto",
distributed_executor_backend: str | None = None,
@@ -116,8 +117,18 @@ def run_test(
)
vllm_outputs_per_mm.append(vllm_output)
hf_runner_kwargs: dict[str, Any] = {}
if model_info.tokenizer:
hf_runner_kwargs["tokenizer_name"] = model_info.tokenizer
if hf_processor is not None:
hf_runner_kwargs["processor"] = hf_processor(model)
hf_model = hf_runner(
model, dtype=dtype, auto_cls=auto_cls, model_kwargs=hf_model_kwargs
model,
dtype=dtype,
auto_cls=auto_cls,
model_kwargs=hf_model_kwargs,
**hf_runner_kwargs,
)
# Some models need to patch things like the model processor, e.g., internvl
@@ -1336,3 +1336,221 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign, assignment]
hf_model.model.generate = patched_generate # type: ignore[method-assign]
return hf_model
def moondream3_processor(model: str):
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
return Moondream3Processor.from_pretrained(model, trust_remote_code=True)
def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
"""Patch HfRunner for Moondream3."""
moondream_processor = hf_model.processor
def processor(*args, text="", images=None, **kwargs):
if images is None:
return moondream_processor(text=text, **kwargs)
images_list = [images] if isinstance(images, Image) else images
return moondream_processor(images=images_list, text=text, **kwargs)
hf_model.processor = processor
# Expose the LM head for logprob extraction.
hf_model.model.get_output_embeddings = lambda: hf_model.model.model.text.lm_head
native_model = hf_model.model.model # MoondreamModel instance
from torch.nn import functional as F
from vllm.model_executor.models.moondream3 import reconstruct_from_crops
# Resolve the placeholder tokens from the tokenizer instead of hard-coding.
image_placeholder_ids = moondream_processor.tokenizer.encode(
"<image>", add_special_tokens=False
)
def _normalize_tiling(tilings):
"""Extract (h, w) tuple from various tiling container formats."""
tiling = tilings
if isinstance(tiling, torch.Tensor):
tiling = tuple(tiling.squeeze().tolist())
elif isinstance(tiling, (list, tuple)):
t0 = tiling[0]
if isinstance(t0, torch.Tensor):
tiling = tuple(t0.tolist())
elif isinstance(t0, (list, tuple)):
tiling = tuple(t0)
return tiling
def _encode_vision(pixel_values, tilings):
"""Run preprocessed crops through vision encoder + projection."""
device = native_model.device
dtype = native_model.vision.pos_emb.dtype
config = native_model.config
pv = pixel_values
while pv.dim() > 4:
pv = pv.squeeze(0)
pv = pv.to(device=device, dtype=dtype)
features = native_model._vis_enc(pv)
grid_size = config.vision.crop_size // config.vision.enc_patch_size
global_feat = features[0]
if features.shape[0] > 1 and tilings is not None:
tiling = _normalize_tiling(tilings)
local = features[1:].view(-1, grid_size, grid_size, config.vision.enc_dim)
reconstructed = reconstruct_from_crops(
local,
tiling,
config.vision.overlap_margin,
patch_size=1,
)
else:
reconstructed = global_feat.view(
grid_size, grid_size, config.vision.enc_dim
)
return native_model._vis_proj(global_feat, reconstructed)
def _find_subsequence(seq, subseq):
"""Find start index of subseq in seq, or None."""
n = len(subseq)
for i in range(len(seq) - n + 1):
if seq[i : i + n] == subseq:
return i
return None
def _generate(
self,
input_ids=None,
pixel_values=None,
tilings=None,
attention_mask=None,
**kwargs,
):
max_new_tokens = kwargs.get("max_new_tokens", 128)
return_dict = kwargs.get("return_dict_in_generate", False)
output_hs = kwargs.get("output_hidden_states", False)
if pixel_values is None:
sequences = input_ids
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=() if output_hs else None,
)
return sequences
# Processor may return lists; extract the single element.
if isinstance(pixel_values, (list, tuple)):
pixel_values = pixel_values[0]
if (
isinstance(tilings, (list, tuple))
and tilings
and not isinstance(tilings[0], int)
):
tilings = tilings[0]
hf_model.model._setup_caches()
native_model.use_flex_decoding = False
device = native_model.device
config = native_model.config
with torch.inference_mode():
for block in native_model.text.blocks:
block.kv_cache.k_cache.zero_()
block.kv_cache.v_cache.zero_()
img_emb = _encode_vision(pixel_values, tilings)
bos_emb = F.embedding(
torch.tensor([[config.tokenizer.bos_id]], device=device),
native_model.text.wte,
)
img_input = torch.cat([bos_emb, img_emb.unsqueeze(0)], dim=1)
prefix_len = img_input.size(1)
mask = native_model.attn_mask[:, :, :prefix_len, :]
pos_ids = torch.arange(prefix_len, dtype=torch.long, device=device)
native_model._prefill(img_input, mask, pos_ids, None)
ids = input_ids.squeeze(0).tolist()
img_start = _find_subsequence(ids, image_placeholder_ids)
if img_start is None:
sequences = input_ids
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=() if output_hs else None,
)
return sequences
prompt_tokens = ids[img_start + len(image_placeholder_ids) :]
if not prompt_tokens:
sequences = input_ids
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=() if output_hs else None,
)
return sequences
prompt_tensor = torch.tensor([prompt_tokens], device=device)
prompt_emb = F.embedding(prompt_tensor, native_model.text.wte)
prompt_len = prompt_emb.size(1)
mask = native_model.attn_mask[:, :, prefix_len : prefix_len + prompt_len, :]
pos_ids = torch.arange(
prefix_len,
prefix_len + prompt_len,
dtype=torch.long,
device=device,
)
hidden = native_model._prefill(prompt_emb, mask, pos_ids, None)
pos = prefix_len + prompt_len
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
logits = native_model.text.lm_head(hidden_last.squeeze(1))
generated = []
all_hidden_states = []
# Record the hidden state that predicted each generated token.
prev_hs = hidden_last
for _ in range(max_new_tokens):
next_token = logits.argmax(dim=-1).item()
if next_token == 0:
break
generated.append(next_token)
if output_hs:
all_hidden_states.append((prev_hs,))
next_emb = F.embedding(
torch.tensor([[next_token]], device=device),
native_model.text.wte,
)
mask = native_model.attn_mask[:, :, pos : pos + 1, :]
pos_ids_step = torch.tensor([pos], dtype=torch.long, device=device)
hidden = native_model._prefill(next_emb, mask, pos_ids_step, None)
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
prev_hs = hidden_last
logits = native_model.text.lm_head(hidden_last.squeeze(1))
pos += 1
result_ids = ids + generated
sequences = torch.tensor([result_ids], device=device)
if return_dict:
return types.SimpleNamespace(
sequences=sequences,
hidden_states=tuple(all_hidden_states) if output_hs else None,
)
return sequences
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
return hf_model
@@ -133,6 +133,7 @@ class VLMTestInfo(NamedTuple):
# Exposed options for HF runner
hf_model_kwargs: dict[str, Any] | None = None
hf_processor: Callable[[str], Any] | None = None
# Indicates we should explicitly pass the EOS from the tokenizer
use_tokenizer_eos: bool = False
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM
@@ -196,6 +197,7 @@ class VLMTestInfo(NamedTuple):
"comparator": self.comparator,
"get_stop_token_ids": self.get_stop_token_ids,
"hf_model_kwargs": self.hf_model_kwargs,
"hf_processor": self.hf_processor,
"stop_str": self.stop_str,
"patch_hf_runner": self.patch_hf_runner,
}
@@ -0,0 +1,553 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Moondream3 multimodal processing.
Includes:
- Processor creation and application tests
- Image tokenization and placeholder expansion tests
- Tiling and cropping logic tests (CPU-based)
- Pixel normalization tests
"""
import numpy as np
import pytest
import torch
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
# Expected multimodal prefix: BOS + 729 image tokens.
EXPECTED_IMAGE_TOKENS = 730
# Vision encoder constants
CROP_SIZE = 378
PATCH_SIZE = 14
MAX_CROPS = 12
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_creation(model_id: str):
"""Test that Moondream3 processor can be created."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
assert processor is not None
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_apply(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that Moondream3 processor can process inputs.
NOTE: The prompt includes the leading BOS token because Moondream3
pre-fills BOS and image embeddings together.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
assert "prompt_token_ids" in processed_inputs
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_pixel_values(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that pixel values are correctly produced."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
# Check mm_kwargs contains pixel_values
mm_kwargs = processed_inputs.get("mm_kwargs")
assert mm_kwargs is not None
mm_data_result = mm_kwargs.get_data()
assert "pixel_values" in mm_data_result
# Verify pixel_values shape
pixel_values = mm_data_result["pixel_values"]
assert pixel_values.dim() == 5 # [batch, num_crops, C, H, W]
assert pixel_values.shape[2] == 3 # RGB channels
assert pixel_values.shape[3] == 378 # crop height
assert pixel_values.shape[4] == 378 # crop width
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_image_token_expansion(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that <image> placeholder is expanded to correct number of tokens."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe.<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_multi_crop_tiling(
model_id: str,
):
"""Test that large images produce correct multi-crop tiling."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
# Create a large image that requires multiple crops
large_image = Image.new("RGB", (1000, 1000), color="blue")
pixel_values, tiling = processor.preprocess_image(large_image)
# Large images should produce more than 1x1 tiling
assert tiling[0] >= 1 and tiling[1] >= 1
# Check that we have global crop + local crops
expected_crops = tiling[0] * tiling[1] + 1
assert pixel_values.shape[0] == expected_crops
@pytest.mark.parametrize(
"image_size",
[
(500, 500),
(800, 600),
(1920, 1080),
],
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_tiling_various_sizes(
image_size: tuple[int, int],
model_id: str,
):
"""Test tiling with various image sizes."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
width, height = image_size
image = Image.new("RGB", (width, height), color="red")
pixel_values, tiling = processor.preprocess_image(image)
# Basic shape checks
assert pixel_values.dim() == 4 # [num_crops, C, H, W]
assert pixel_values.shape[1] == 3 # RGB
assert pixel_values.shape[2] == 378 # crop height
assert pixel_values.shape[3] == 378 # crop width
# Tiling should respect max_crops (12)
assert tiling[0] * tiling[1] <= 12
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_pixel_normalization(
model_id: str,
):
"""Test that pixel values are normalized to [-1, 1] range."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
# Create test image
image = Image.new("RGB", (378, 378), color="green")
pixel_values, _ = processor.preprocess_image(image)
# Normalization: (x - 0.5) / 0.5 = 2*x - 1
# For input [0, 1], output should be [-1, 1]
assert pixel_values.min() >= -1.0
assert pixel_values.max() <= 1.0
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_chat_template_with_image(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that chat template correctly formats BOS + image + prompt."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = ctx.tokenizer
# Use the chat template format
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
token_ids = processed_inputs["prompt_token_ids"]
# BOS token (<|endoftext|>) should be token ID 0
bos_token_id = tokenizer.encode("<|endoftext|>", add_special_tokens=False)[0]
assert bos_token_id == 0
# First token should be BOS
assert token_ids[0] == bos_token_id
@pytest.mark.parametrize(
"content",
[
pytest.param(
[
{
"type": "image_url",
"image_url": {"url": "https://example.invalid/image.png"},
},
{"type": "text", "text": "What is in this image?"},
],
id="image-first",
),
pytest.param(
[
{"type": "text", "text": "What is in this image?"},
{
"type": "image_url",
"image_url": {"url": "https://example.invalid/image.png"},
},
],
id="text-first",
),
],
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_chat_template_content_list_uses_moondream_image_prefix(
image_assets: ImageTestAssets,
content: list[dict[str, object]],
model_id: str,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor()
prompt = hf_processor.tokenizer.apply_chat_template(
[{"role": "user", "content": content}],
chat_template=hf_processor.chat_template,
tokenize=False,
)
expected_prompt = (
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
"What is in this image?<|md_reserved_2|>"
)
assert prompt == expected_prompt
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data({"image": [image_assets[0].pil_image]}),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_bos_token_always_first(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that BOS token (ID 0) is always at position 0."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Start with BOS token explicitly
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe this image.<|md_reserved_2|>" # noqa: E501
mm_data = {"image": [image_assets[0].pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
token_ids = processed_inputs["prompt_token_ids"]
# Token ID 0 (<|endoftext|>) should be the first token
assert token_ids[0] == 0, (
f"Expected BOS token (0) at position 0, got {token_ids[0]}"
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_with_small_image(
model_id: str,
):
"""Test processor with image smaller than crop size."""
from PIL import Image
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
# Small image (smaller than crop size)
small_image = Image.new("RGB", (100, 100), color="yellow")
pixel_values, tiling = processor.preprocess_image(small_image)
# Small images should use 1x1 tiling
assert tiling == (1, 1)
# Should have 2 crops (global + 1 local)
assert pixel_values.shape[0] == 2
@pytest.mark.parametrize(
"image_kind",
[
pytest.param("numpy_hwc", id="numpy-hwc"),
pytest.param("numpy_chw", id="numpy-chw"),
pytest.param("torch_chw", id="torch-chw"),
],
)
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_preprocess_image_accepts_non_pil_inputs(
image_assets: ImageTestAssets,
image_kind: str,
model_id: str,
):
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
pil_image = image_assets[0].pil_image.convert("RGB")
hwc_array = np.asarray(pil_image)
expected_pixel_values, expected_tiling = processor.preprocess_image(pil_image)
if image_kind == "numpy_hwc":
image = hwc_array
elif image_kind == "numpy_chw":
image = np.transpose(hwc_array, (2, 0, 1))
else:
image = torch.from_numpy(np.transpose(hwc_array, (2, 0, 1)).copy())
pixel_values, tiling = processor.preprocess_image(image)
assert tiling == expected_tiling
assert pixel_values.shape == expected_pixel_values.shape
assert pixel_values.dtype == torch.bfloat16
assert torch.equal(pixel_values, expected_pixel_values)
@pytest.mark.parametrize("image_kind", ["numpy_chw", "torch_chw"])
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
def test_processor_apply_accepts_non_pil_image_inputs(
image_assets: ImageTestAssets,
image_kind: str,
model_id: str,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
hwc_array = np.asarray(image_assets[0].pil_image.convert("RGB"))
chw_array = np.transpose(hwc_array, (2, 0, 1)).copy()
image = chw_array if image_kind == "numpy_chw" else torch.from_numpy(chw_array)
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data({"image": [image]}),
hf_processor_mm_kwargs={},
)
image_placeholders = processed_inputs["mm_placeholders"]["image"]
assert len(image_placeholders) == 1
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
mm_kwargs = processed_inputs["mm_kwargs"].get_data()
assert mm_kwargs["pixel_values"].shape[2:] == (3, 378, 378)
class TestMoondream3TilingLogic:
"""CPU-based tests for Moondream3 tiling selection logic.
These tests validate the select_tiling() function which determines
how images are divided into crops for the vision encoder.
"""
def test_small_image_no_tiling(self):
"""Small images should use 1x1 tiling."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=300, width=300, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
assert tiling == (1, 1)
def test_exact_crop_size(self):
"""Image exactly at crop size should use 1x1."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=CROP_SIZE, width=CROP_SIZE, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
assert tiling == (1, 1)
def test_large_square_image(self):
"""Large square image should use multiple tiles."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=800, width=800, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
h_tiles, w_tiles = tiling
assert h_tiles >= 2
assert w_tiles >= 2
assert h_tiles * w_tiles <= MAX_CROPS
def test_wide_image(self):
"""Wide image should have more width tiles."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=400, width=1200, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
h_tiles, w_tiles = tiling
assert w_tiles >= h_tiles
def test_tall_image(self):
"""Tall image should have more height tiles."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=1200, width=400, crop_size=CROP_SIZE, max_crops=MAX_CROPS
)
h_tiles, w_tiles = tiling
assert h_tiles >= w_tiles
def test_respects_max_crops(self):
"""Tiling should not exceed max_crops."""
from vllm.transformers_utils.processors.moondream3 import select_tiling
tiling = select_tiling(
height=2000, width=2000, crop_size=CROP_SIZE, max_crops=4
)
h_tiles, w_tiles = tiling
assert h_tiles * w_tiles <= 4
class TestMoondream3VisionShapes:
"""CPU-based tests for vision encoder expected shapes.
These tests verify the mathematical relationships between
crop size, patch size, and token counts.
"""
def test_expected_patch_count(self):
"""Test 378/14 = 27 patches per side, 729 total."""
patches_per_side = CROP_SIZE // PATCH_SIZE
total_patches = patches_per_side**2
assert patches_per_side == 27
assert total_patches == EXPECTED_IMAGE_TOKENS - 1
def test_patch_embedding_input_dim(self):
"""Test patch embedding input dimension."""
channels = 3
input_dim = PATCH_SIZE * PATCH_SIZE * channels
assert input_dim == 14 * 14 * 3
assert input_dim == 588
class TestMoondream3TauAttention:
"""CPU-based tests for tau attention scaling components.
These tests validate the tau attention formula used in Moondream3:
- Token-based: tok_q = tanh(gelu(qkv) @ tau_wq.T)
- Position-based: tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5)
"""
def test_tau_position_range(self):
"""Test tau position scaling produces values in valid range."""
num_heads = 32
seq_len = 100
tau_alpha = torch.randn(num_heads)
positions = torch.arange(seq_len)
pos_float = (positions.float() + 1.0).clamp(min=1e-6)
pos_log = pos_float.log()
tau_pos = 1.0 + (torch.sigmoid(tau_alpha[:, None] * pos_log[None, :]) - 0.5)
assert tau_pos.shape == (num_heads, seq_len)
# tau_pos should be between 0.5 and 1.5
assert tau_pos.min() >= 0.5
assert tau_pos.max() <= 1.5
def test_tau_token_output_range(self):
"""Test tau token scaling output is bounded by tanh."""
import torch.nn.functional as F
seq_len = 100
qkv_dim = 6144 # 2048 * 3
num_heads = 32
qkv = torch.randn(seq_len, qkv_dim)
tau_wq = torch.randn(num_heads, qkv_dim)
tok_feat = F.gelu(qkv)
tok_q = torch.tanh(tok_feat @ tau_wq.t())
assert tok_q.shape == (seq_len, num_heads)
# tanh output is bounded by [-1, 1]
assert tok_q.min() >= -1.0
assert tok_q.max() <= 1.0
+10
View File
@@ -1122,6 +1122,16 @@ _MULTIMODAL_EXAMPLE_MODELS = {
extras={"olmo": "allenai/Molmo-7B-O-0924"},
trust_remote_code=True,
),
"Moondream3ForCausalLM": _HfExamplesInfo(
"moondream/moondream3-preview",
tokenizer="moondream/starmie-v1",
trust_remote_code=True,
),
"HfMoondream": _HfExamplesInfo(
"moondream/moondream3-preview",
tokenizer="moondream/starmie-v1",
trust_remote_code=True,
),
"Molmo2ForConditionalGeneration": _HfExamplesInfo(
"allenai/Molmo2-8B",
extras={"olmo": "allenai/Molmo2-O-7B"},
+14
View File
@@ -523,6 +523,17 @@ def dummy_hf_overrides(
text_config.update(update_dict)
# Update n_layers and moe configs for Moondream3 model
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
text_config.update(
{
"n_layers": num_hidden_layers,
"moe_num_experts": num_experts,
"moe_experts_per_token": 2,
"moe_start_layer": num_hidden_layers,
}
)
if hasattr(hf_config, "vision_config"):
hf_config.vision_config.update(
{
@@ -531,6 +542,9 @@ def dummy_hf_overrides(
}
)
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
hf_config.vision_config.update({"enc_n_layers": 1})
# e.g.: ibm-granite/granite-speech-3.3-2b
if hasattr(hf_config, "encoder_config"):
hf_config.encoder_config.update(
File diff suppressed because it is too large Load Diff
+2
View File
@@ -484,6 +484,8 @@ _MULTIMODAL_MODELS = {
),
"MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
"Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
"Moondream3ForCausalLM": ("moondream3", "Moondream3ForCausalLM"),
"HfMoondream": ("moondream3", "Moondream3ForCausalLM"),
"NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
+1
View File
@@ -114,6 +114,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
mlp_speculator="MLPSpeculatorConfig",
medusa="MedusaConfig",
midashenglm="MiDashengLMConfig",
moondream3="Moondream3Config",
eagle="EAGLEConfig",
speculators="SpeculatorsConfig",
nemotron="NemotronConfig",
@@ -50,6 +50,9 @@ _CLASS_TO_MODULE: dict[str, str] = {
"MedusaConfig": "vllm.transformers_utils.configs.medusa",
"MiDashengLMConfig": "vllm.transformers_utils.configs.midashenglm",
"MLPSpeculatorConfig": "vllm.transformers_utils.configs.mlp_speculator",
"Moondream3Config": "vllm.transformers_utils.configs.moondream3",
"Moondream3TextConfig": "vllm.transformers_utils.configs.moondream3",
"Moondream3VisionConfig": "vllm.transformers_utils.configs.moondream3",
"MoonViTConfig": "vllm.transformers_utils.configs.moonvit",
"KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear",
"KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl",
@@ -111,6 +114,9 @@ __all__ = [
"MedusaConfig",
"MiDashengLMConfig",
"MLPSpeculatorConfig",
"Moondream3Config",
"Moondream3TextConfig",
"Moondream3VisionConfig",
"MoonViTConfig",
"KimiLinearConfig",
"KimiVLConfig",
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration for Moondream3 model."""
from transformers import PretrainedConfig
class Moondream3VisionConfig(PretrainedConfig):
"""Vision encoder configuration for Moondream3."""
model_type = "moondream3_vision"
def __init__(
self,
enc_dim: int = 1152,
enc_patch_size: int = 14,
enc_n_layers: int = 27,
enc_ff_dim: int = 4304,
enc_n_heads: int = 16,
proj_inner_dim: int = 8192,
crop_size: int = 378,
max_crops: int = 12,
overlap_margin: int = 4,
**kwargs,
):
super().__init__(**kwargs)
self.enc_dim = enc_dim
self.enc_patch_size = enc_patch_size
self.enc_n_layers = enc_n_layers
self.enc_ff_dim = enc_ff_dim
self.enc_n_heads = enc_n_heads
self.proj_inner_dim = proj_inner_dim
self.crop_size = crop_size
self.max_crops = max_crops
self.overlap_margin = overlap_margin
# Standard HuggingFace attributes for vision config
self.hidden_size = enc_dim
self.num_attention_heads = enc_n_heads
self.num_hidden_layers = enc_n_layers
self.intermediate_size = enc_ff_dim
self.patch_size = enc_patch_size
self.image_size = crop_size
class Moondream3TextConfig(PretrainedConfig):
"""Text decoder configuration for Moondream3."""
model_type = "moondream3_text"
def __init__(
self,
dim: int = 2048,
ff_dim: int = 8192,
n_layers: int = 24,
vocab_size: int = 51200,
max_context: int = 4096,
n_heads: int = 32,
n_kv_heads: int = 32,
prefix_attn: int = 730,
rope_theta: float = 1500000.0,
moe: dict | None = None,
**kwargs,
):
super().__init__(**kwargs)
# Store original moondream3 config names
self.dim = dim
self.ff_dim = ff_dim
self.n_layers = n_layers
self.n_heads = n_heads
self.n_kv_heads = n_kv_heads
self.prefix_attn = prefix_attn
self.max_context = max_context
self.rope_theta = rope_theta
# MoE config
moe = moe or {}
self.moe_start_layer = moe.get("start_layer", 4)
self.moe_num_experts = moe.get("n_experts", 64)
self.moe_experts_per_token = moe.get("n_experts_per_tok", 8)
self.moe_expert_inner_dim = moe.get("expert_inner_dim", 1024)
# Standard HuggingFace attributes (required by vLLM)
self.hidden_size = dim
self.num_attention_heads = n_heads
self.num_key_value_heads = n_kv_heads
self.num_hidden_layers = n_layers
self.intermediate_size = ff_dim
self.vocab_size = vocab_size
self.max_position_embeddings = max_context
# Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS.
# Token 3 (<|md_reserved_2|>) is an answer delimiter that the model
# implementation suppresses during generation.
self.bos_token_id = 0
self.eos_token_id = 0
# MoE standard attributes
self.num_local_experts = self.moe_num_experts
self.num_experts_per_tok = self.moe_experts_per_token
class Moondream3Config(PretrainedConfig):
"""Combined configuration for Moondream3 multimodal model."""
model_type = "moondream3"
is_composition = True
def __init__(
self,
config: dict | None = None,
**kwargs,
):
config = config or {}
# Parse text config
text_config = config.get("text", {})
self.text_config: Moondream3TextConfig = Moondream3TextConfig(**text_config)
# Parse vision config
vision_config = config.get("vision", {})
self.vision_config = Moondream3VisionConfig(**vision_config)
# Store the original config dict for model access
self.config = config
tokenizer_config = config.get("tokenizer", {})
self.answer_token_id = tokenizer_config.get("answer_id", 3)
super().__init__(**kwargs)
# Expose key attributes at top level for vLLM compatibility
self.hidden_size = self.text_config.hidden_size
self.num_attention_heads = self.text_config.num_attention_heads
self.num_key_value_heads = self.text_config.num_key_value_heads
self.num_hidden_layers = self.text_config.num_hidden_layers
self.vocab_size = self.text_config.vocab_size
self.intermediate_size = self.text_config.intermediate_size
# Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS.
# Token 3 (<|md_reserved_2|>) is an answer delimiter that the model
# implementation suppresses during generation.
self.bos_token_id = 0
self.eos_token_id = 0
def get_text_config(self, decoder: bool = False) -> "Moondream3TextConfig":
"""Return the text config for vLLM's text_config detection.
Args:
decoder: Ignored. Only used for encoder-decoder models.
"""
return self.text_config
@@ -267,6 +267,7 @@ class ModelArchConfigConvertorBase:
"bagel",
"gemma3",
"molmo2",
"moondream3",
"paligemma",
"umm",
)
@@ -23,6 +23,7 @@ __all__ = [
"H2OVLProcessor",
"HunYuanVLProcessor",
"HunYuanVLImageProcessor",
"Moondream3Processor",
"InternVLProcessor",
"IsaacProcessor",
"KimiAudioProcessor",
@@ -61,6 +62,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"MiMoOmniProcessor": "vllm.transformers_utils.processors.mimo_v2_omni",
"MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
"MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
"Moondream3Processor": "vllm.transformers_utils.processors.moondream3",
"NanoNemotronVLProcessor": "vllm.transformers_utils.processors.nano_nemotron_vl",
"NemotronVLProcessor": "vllm.transformers_utils.processors.nemotron_vl",
"LlamaNemotronVLEmbedProcessor": "vllm.transformers_utils.processors.nemotron_vl",
@@ -0,0 +1,541 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Custom processor for Moondream3 model."""
import math
import numpy as np
import torch
from PIL import Image
from transformers import AutoProcessor, BatchFeature
from transformers.image_utils import ImageInput
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
from transformers.tokenization_utils_base import (
PreTokenizedInput,
PreTrainedTokenizerBase,
TextInput,
)
from vllm.multimodal.image import convert_image_mode
__all__ = ["Moondream3Processor"]
class Moondream3ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
_defaults = {
"text_kwargs": {
"padding": False,
},
"images_kwargs": {
"max_crops": 12,
"overlap_margin": 4,
"crop_size": 378,
"patch_size": 14,
"convert_to_rgb": True,
"return_tensors": "pt",
},
}
def select_tiling(
height: int, width: int, crop_size: int, max_crops: int
) -> tuple[int, int]:
"""Determine the optimal number of tiles to cover an image."""
if height <= crop_size or width <= crop_size:
return (1, 1)
min_h = math.ceil(height / crop_size)
min_w = math.ceil(width / crop_size)
if min_h * min_w > max_crops:
ratio = math.sqrt(max_crops / (min_h * min_w))
return (max(1, math.floor(min_h * ratio)), max(1, math.floor(min_w * ratio)))
h_tiles = math.floor(math.sqrt(max_crops * height / width))
w_tiles = math.floor(math.sqrt(max_crops * width / height))
h_tiles = max(h_tiles, min_h)
w_tiles = max(w_tiles, min_w)
if h_tiles * w_tiles > max_crops:
if w_tiles > h_tiles:
w_tiles = math.floor(max_crops / h_tiles)
else:
h_tiles = math.floor(max_crops / w_tiles)
return (max(1, h_tiles), max(1, w_tiles))
class Moondream3Processor(ProcessorMixin):
"""
Constructs a Moondream3 processor which handles image preprocessing
and tokenization for the Moondream3 multimodal model.
Args:
tokenizer: The tokenizer to use for text processing.
chat_template: Optional chat template string.
crop_size: Size of each image crop.
max_crops: Maximum number of crops per image.
overlap_margin: Margin for overlapping crops in patches.
patch_size: Size of each patch.
"""
attributes = ["tokenizer"]
valid_kwargs = [
"chat_template",
"crop_size",
"max_crops",
"overlap_margin",
"patch_size",
]
tokenizer_class = "AutoTokenizer"
# Use separate tokenizer repo
_tokenizer_repo = "moondream/starmie-v1"
# Default chat template for Moondream3
# Moondream uses special tokens for prompting:
# - Token 0 (<|endoftext|>): BOS token (ALWAYS present at position 0)
# - Token 1 (<|md_reserved_0|>): Start of instruction
# - Token 2 (<|md_reserved_1|>): Separator before question
# - Token 3 (<|md_reserved_2|>): End of question / start of answer
#
# Task routing based on text prefix:
# "caption [short|normal|long]" → describe<|md_reserved_1|>{length}
# "describe [short|normal|long]" → describe<|md_reserved_1|>{length}
# otherwise → query<|md_reserved_1|><text>
#
# Format with image:
# <|endoftext|><image><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|>
# Format without image:
# <|endoftext|><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|>
_default_chat_template = (
"{% for message in messages %}"
"{% if message['role'] == 'user' %}"
"{% if message['content'] is string %}"
# Simple string content (with image assumed) - route by prefix
"<|endoftext|><image><|md_reserved_0|>"
"{% if message['content'] == 'caption' %}"
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
"{% elif message['content'].startswith('caption ') %}"
"describe<|md_reserved_1|>{{ message['content'][8:] }}<|md_reserved_2|>"
"{% elif message['content'] == 'describe' %}"
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
"{% elif message['content'].startswith('describe ') %}"
"describe<|md_reserved_1|>{{ message['content'][9:] }}<|md_reserved_2|>"
"{% else %}"
"query<|md_reserved_1|>{{ message['content'] }}<|md_reserved_2|>"
"{% endif %}"
"{% else %}"
# List content - build Moondream's image prefix independently of
# OpenAI-style content part order, then render the text task.
"<|endoftext|>"
"{% for content in message['content'] %}"
"{% if content['type'] in ['image', 'image_url', 'input_image', 'image_pil'] %}" # noqa: E501
"<image>"
"{% endif %}"
"{% endfor %}"
"{% for content in message['content'] %}"
"{% if content['type'] == 'text' %}"
"<|md_reserved_0|>"
"{% if content['text'] == 'caption' %}"
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
"{% elif content['text'].startswith('caption ') %}"
"describe<|md_reserved_1|>{{ content['text'][8:] }}<|md_reserved_2|>"
"{% elif content['text'] == 'describe' %}"
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
"{% elif content['text'].startswith('describe ') %}"
"describe<|md_reserved_1|>{{ content['text'][9:] }}<|md_reserved_2|>"
"{% else %}"
"query<|md_reserved_1|>{{ content['text'] }}<|md_reserved_2|>"
"{% endif %}"
"{% endif %}"
"{% endfor %}"
"{% endif %}"
"{% elif message['role'] == 'assistant' %}"
"{{ message['content'] }}"
"{% endif %}"
"{% endfor %}"
)
def __init__(
self,
tokenizer: PreTrainedTokenizerBase | None = None,
chat_template: str | None = None,
crop_size: int = 378,
max_crops: int = 12,
overlap_margin: int = 4,
patch_size: int = 14,
**kwargs,
):
self.image_token = "<image>"
self.crop_size = crop_size
self.max_crops = max_crops
self.overlap_margin = overlap_margin
self.patch_size = patch_size
# Number of patches per crop (27x27 = 729 for 378/14)
self.patches_per_crop = (crop_size // patch_size) ** 2
# Use default chat template if none provided
if chat_template is None:
chat_template = self._default_chat_template
super().__init__(tokenizer, chat_template=chat_template)
@classmethod
def from_pretrained(
cls,
pretrained_model_name_or_path,
**kwargs,
):
"""
Load the processor, using a separate tokenizer repo.
The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1'
instead of having tokenizer files in the model repo.
"""
from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.utils import cached_file
tokenizer = kwargs.pop("tokenizer", None)
tokenizer_kwargs = {
"trust_remote_code": kwargs.get("trust_remote_code", False),
}
for key in (
"cache_dir",
"force_download",
"local_files_only",
"revision",
"subfolder",
"token",
"use_fast",
):
if key in kwargs:
tokenizer_kwargs[key] = kwargs[key]
cached_file_kwargs = {
key: tokenizer_kwargs[key]
for key in (
"cache_dir",
"force_download",
"local_files_only",
"revision",
"subfolder",
"token",
)
if key in tokenizer_kwargs
}
def load_tokenizer(repo_or_path):
try:
return AutoTokenizer.from_pretrained(repo_or_path, **tokenizer_kwargs)
except Exception:
tokenizer_file = cached_file(
repo_or_path,
"tokenizer.json",
**cached_file_kwargs,
)
return PreTrainedTokenizerFast(
tokenizer_file=tokenizer_file,
clean_up_tokenization_spaces=False,
)
if isinstance(tokenizer, str):
tokenizer = load_tokenizer(tokenizer)
if tokenizer is None:
# Prefer model-local tokenizer files first. If unavailable, fall
# back to moondream's dedicated tokenizer repository.
try:
tokenizer = load_tokenizer(pretrained_model_name_or_path)
except Exception:
tokenizer = load_tokenizer(cls._tokenizer_repo)
# Configure special tokens for Moondream3
# BOS and EOS are both token 0 (<|endoftext|>), matching the native
# config (TokenizerConfig.bos_id=0, eos_id=0). This is standard for
# GPT-2 style models where <|endoftext|> signals both start and end.
# Token 1 (<|md_reserved_0|>) is a template delimiter, NOT the EOS.
tokenizer.bos_token = "<|endoftext|>"
tokenizer.bos_token_id = 0
tokenizer.eos_token = "<|endoftext|>"
tokenizer.eos_token_id = 0
# Extract processor-specific kwargs
crop_size = kwargs.pop("crop_size", 378)
max_crops = kwargs.pop("max_crops", 12)
overlap_margin = kwargs.pop("overlap_margin", 4)
patch_size = kwargs.pop("patch_size", 14)
chat_template = kwargs.pop("chat_template", None)
# Set default chat template on tokenizer if not already set
if chat_template is None:
chat_template = cls._default_chat_template
if tokenizer.chat_template is None:
tokenizer.chat_template = chat_template
return cls(
tokenizer=tokenizer,
chat_template=chat_template,
crop_size=crop_size,
max_crops=max_crops,
overlap_margin=overlap_margin,
patch_size=patch_size,
)
def __call__(
self,
images: ImageInput = None,
text: TextInput
| PreTokenizedInput
| list[TextInput]
| list[PreTokenizedInput] = None,
**kwargs: Unpack[Moondream3ProcessorKwargs],
) -> BatchFeature:
"""
Process images and text for Moondream3 model.
Args:
images: Input images (PIL Image, numpy array, or list thereof).
text: Input text or list of texts.
**kwargs: Additional processing arguments.
Returns:
BatchFeature with processed inputs.
"""
output_kwargs = self._merge_kwargs(
Moondream3ProcessorKwargs,
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
**kwargs,
)
# Process images
image_features = {}
if images is not None:
processed_images = []
tilings = []
images_list = images if isinstance(images, list) else [images]
for image in images_list:
pixel_values, tiling = self.preprocess_image(
image, **output_kwargs["images_kwargs"]
)
processed_images.append(pixel_values)
tilings.append(tiling)
if processed_images:
image_features["pixel_values"] = processed_images
image_features["tilings"] = tilings
# Process text
if text is not None:
if not isinstance(text, list):
text = [text]
# Get text kwargs, remove keys we set ourselves
text_kwargs = output_kwargs.get("text_kwargs", {}).copy()
text_kwargs.pop("return_tensors", None)
text_kwargs.pop("add_special_tokens", None)
# Tokenize text
tokenized = self.tokenizer(
text,
add_special_tokens=True,
return_tensors="pt",
**text_kwargs,
)
output = BatchFeature(data=dict(tokenized))
# Add image features
if image_features:
output["pixel_values"] = image_features["pixel_values"]
output["tilings"] = image_features["tilings"]
return output
# If only images were provided
return BatchFeature(data=image_features)
@staticmethod
def _image_array_to_uint8(array: np.ndarray) -> np.ndarray:
if array.dtype == np.uint8:
return np.ascontiguousarray(array)
if array.dtype == np.bool_:
return np.ascontiguousarray(array.astype(np.uint8) * 255)
if np.issubdtype(array.dtype, np.floating):
array = np.nan_to_num(array, nan=0.0, posinf=255.0, neginf=0.0)
if array.size > 0 and array.max() <= 1.0:
array = array * 255.0
array = np.rint(array)
return np.ascontiguousarray(np.clip(array, 0, 255).astype(np.uint8))
@staticmethod
def _to_pil_image(image: ImageInput) -> Image.Image:
if isinstance(image, Image.Image):
return image
if isinstance(image, torch.Tensor):
tensor = image.detach().cpu()
if tensor.dtype == torch.bfloat16:
tensor = tensor.to(torch.float32)
image_array = tensor.numpy()
elif isinstance(image, np.ndarray):
image_array = image
else:
raise TypeError(
"Moondream3 images must be PIL images, numpy arrays, "
f"or torch tensors, got {type(image)!r}."
)
if image_array.ndim == 2:
image_array = Moondream3Processor._image_array_to_uint8(image_array)
return Image.fromarray(image_array)
if image_array.ndim != 3:
raise ValueError(
"Moondream3 image arrays must have 2 or 3 dimensions, "
f"got shape {image_array.shape}."
)
channel_dims = (1, 3, 4)
if image_array.shape[-1] not in channel_dims:
if image_array.shape[0] not in channel_dims:
raise ValueError(
"Moondream3 image arrays must be HWC or CHW with 1, 3, "
f"or 4 channels, got shape {image_array.shape}."
)
image_array = np.transpose(image_array, (1, 2, 0))
image_array = Moondream3Processor._image_array_to_uint8(image_array)
if image_array.shape[-1] == 1:
image_array = image_array[..., 0]
return Image.fromarray(image_array)
def preprocess_image(
self,
image: ImageInput,
max_crops: int = 12,
overlap_margin: int = 4,
crop_size: int = 378,
patch_size: int = 14,
convert_to_rgb: bool = True,
return_tensors: str = "pt",
) -> tuple[torch.Tensor, tuple[int, int]]:
"""
Preprocess an image using overlap-and-resize cropping strategy.
Args:
image: Input PIL image, numpy array, or torch tensor.
max_crops: Maximum number of crops.
overlap_margin: Margin for overlapping in patches.
crop_size: Size of each crop.
patch_size: Size of each patch.
convert_to_rgb: Whether to convert to RGB.
return_tensors: Return type ("pt" for PyTorch).
Returns:
Tuple of (pixel_values tensor, tiling tuple).
"""
image = self._to_pil_image(image)
if convert_to_rgb:
image = convert_image_mode(image, "RGB")
# Convert to numpy array
image_array = np.array(image)
original_h, original_w = image_array.shape[:2]
margin_pixels = patch_size * overlap_margin
total_margin_pixels = margin_pixels * 2
crop_patches = crop_size // patch_size
crop_window_patches = crop_patches - (2 * overlap_margin)
crop_window_size = crop_window_patches * patch_size
tiling = select_tiling(
original_h - total_margin_pixels,
original_w - total_margin_pixels,
crop_window_size,
max_crops,
)
n_crops = tiling[0] * tiling[1] + 1
crops = np.zeros((n_crops, crop_size, crop_size, 3), dtype=np.uint8)
target_size = (
tiling[0] * crop_window_size + total_margin_pixels,
tiling[1] * crop_window_size + total_margin_pixels,
)
# Resize image
pil_img = Image.fromarray(image_array)
resized = pil_img.resize(
(int(target_size[1]), int(target_size[0])),
resample=Image.Resampling.LANCZOS,
)
resized_array = np.asarray(resized)
# Create global crop
global_pil = pil_img.resize(
(crop_size, crop_size), resample=Image.Resampling.LANCZOS
)
crops[0] = np.asarray(global_pil)
# Create local crops
for i in range(tiling[0]):
for j in range(tiling[1]):
y0 = i * crop_window_size
x0 = j * crop_window_size
y_end = min(y0 + crop_size, resized_array.shape[0])
x_end = min(x0 + crop_size, resized_array.shape[1])
crop_region = resized_array[y0:y_end, x0:x_end]
crop_idx = 1 + i * tiling[1] + j
h_slice = slice(None, crop_region.shape[0])
w_slice = slice(None, crop_region.shape[1])
crops[crop_idx, h_slice, w_slice] = crop_region
# Convert to tensor: (n_crops, H, W, C) -> (n_crops, C, H, W)
pixel_values = np.transpose(crops, (0, 3, 1, 2))
if return_tensors == "pt":
# Match HF reference preprocessing exactly: convert uint8 crops to
# bfloat16 before in-place normalization.
pixel_values = (
torch.from_numpy(pixel_values)
.to(dtype=torch.bfloat16)
.div_(255.0)
.sub_(0.5)
.div_(0.5)
)
else:
pixel_values = pixel_values.astype(np.float32) / 255.0
pixel_values = (pixel_values - 0.5) / 0.5
return pixel_values, tiling
def get_num_image_tokens(self) -> int:
"""Return the number of image tokens (729 = 27x27 patches)."""
return self.patches_per_crop
def batch_decode(self, *args, **kwargs):
"""Forward to tokenizer's batch_decode."""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""Forward to tokenizer's decode."""
return self.tokenizer.decode(*args, **kwargs)
@property
def model_input_names(self):
tokenizer_input_names = self.tokenizer.model_input_names
return tokenizer_input_names + ["pixel_values", "tilings"]
AutoProcessor.register("Moondream3Processor", Moondream3Processor)