[Model] Add Moondream3 model support(only query and caption skills) (#32325)
Signed-off-by: Dong Wang <dongw2019@gmail.com>
This commit is contained in:
@@ -215,6 +215,67 @@ When loading RGBA images (images with transparency), vLLM converts them to RGB f
|
||||
- This setting only affects RGBA images with transparency; RGB images are unchanged
|
||||
- If not specified, the default white background `(255, 255, 255)` is used for backward compatibility
|
||||
|
||||
#### Moondream3 Prompt Recipes { #moondream3-prompt-recipes }
|
||||
|
||||
`Moondream3ForCausalLM` supports two task-specific prompt formats:
|
||||
|
||||
- `query`: ask a question about the image.
|
||||
- `caption`: generate a caption for the image.
|
||||
|
||||
```python
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
|
||||
llm = LLM(
|
||||
model="moondream/moondream3-preview",
|
||||
tokenizer="moondream/starmie-v1",
|
||||
trust_remote_code=True,
|
||||
max_model_len=2048,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
|
||||
image = ImageAsset("stop_sign").pil_image
|
||||
|
||||
|
||||
def make_query_prompt(question: str) -> str:
|
||||
return (
|
||||
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
|
||||
f"{question}<|md_reserved_2|>"
|
||||
)
|
||||
|
||||
|
||||
def make_caption_prompt(length: str = "normal") -> str:
|
||||
return (
|
||||
"<|endoftext|><image><|md_reserved_0|>"
|
||||
f"describe<|md_reserved_1|>{length}<|md_reserved_2|>"
|
||||
)
|
||||
|
||||
|
||||
query_out = llm.generate(
|
||||
{
|
||||
"prompt": make_query_prompt("What is shown in this image?"),
|
||||
"multi_modal_data": {"image": image},
|
||||
},
|
||||
SamplingParams(max_tokens=64, temperature=0),
|
||||
)[0].outputs[0].text
|
||||
|
||||
caption_out = llm.generate(
|
||||
{
|
||||
"prompt": make_caption_prompt(),
|
||||
"multi_modal_data": {"image": image},
|
||||
},
|
||||
SamplingParams(max_tokens=100, temperature=0),
|
||||
)[0].outputs[0].text
|
||||
|
||||
print("query:", query_out)
|
||||
print("caption:", caption_out)
|
||||
```
|
||||
|
||||
!!! note
|
||||
The native Moondream3 model also has `detect` and `point` skills. Those
|
||||
require custom coordinate decoding and are not exposed by this vLLM
|
||||
implementation.
|
||||
|
||||
### Video Inputs
|
||||
|
||||
You can pass a list of NumPy arrays directly to the `'video'` field of the multi-modal dictionary
|
||||
|
||||
@@ -599,6 +599,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I<sup>+</sup> | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ |
|
||||
| `MolmoForCausalLM` | Molmo | T + I<sup>+</sup> | `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc. | ✅︎ | ✅︎ |
|
||||
| `Molmo2ForConditionalGeneration` | Molmo2 | T + I<sup>+</sup> / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B` | ✅︎ | ✅︎ |
|
||||
| `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |
|
||||
| `MusicFlamingoForConditionalGeneration` | MusicFlamingo | T + A | `nvidia/music-flamingo-2601-hf`, `nvidia/music-flamingo-think-2601-hf` | ✅︎ | ✅︎ |
|
||||
| `NVLM_D_Model` | NVLM-D 1.0 | T + I<sup>+</sup> | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
|
||||
| `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + I<sup>E+</sup> | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |
|
||||
@@ -661,6 +662,12 @@ Some models are supported only via the [Transformers modeling backend](#transfor
|
||||
!!! note
|
||||
For `InternVLChatModel`, only InternVL2.5 with Qwen2.5 text backbone (`OpenGVLab/InternVL2.5-1B` etc.), InternVL3 and InternVL3.5 have video inputs support currently.
|
||||
|
||||
!!! note
|
||||
`Moondream3ForCausalLM` uses task-specific prompt templates for `query`
|
||||
and `caption`. The native `detect` and `point` skills require custom
|
||||
coordinate decoding and are not exposed by this vLLM implementation.
|
||||
See [Moondream3 prompt recipes](../features/multimodal_inputs.md#moondream3-prompt-recipes).
|
||||
|
||||
!!! note
|
||||
To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
|
||||
|
||||
|
||||
+31
-8
@@ -371,6 +371,8 @@ class HfRunner:
|
||||
is_cross_encoder: bool = False,
|
||||
skip_tokenizer_init: bool = False,
|
||||
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
|
||||
tokenizer_name: str | None = None,
|
||||
processor: Any | None = None,
|
||||
# Set this to avoid hanging issue
|
||||
default_torch_num_threads: int | None = None,
|
||||
) -> None:
|
||||
@@ -391,6 +393,8 @@ class HfRunner:
|
||||
is_cross_encoder=is_cross_encoder,
|
||||
skip_tokenizer_init=skip_tokenizer_init,
|
||||
auto_cls=auto_cls,
|
||||
tokenizer_name=tokenizer_name,
|
||||
processor=processor,
|
||||
)
|
||||
|
||||
def _init(
|
||||
@@ -405,6 +409,8 @@ class HfRunner:
|
||||
is_cross_encoder: bool = False,
|
||||
skip_tokenizer_init: bool = False,
|
||||
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
|
||||
tokenizer_name: str | None = None,
|
||||
processor: Any | None = None,
|
||||
) -> None:
|
||||
model_name = maybe_model_redirect(model_name)
|
||||
self.model_name = model_name
|
||||
@@ -484,20 +490,27 @@ class HfRunner:
|
||||
if not skip_tokenizer_init:
|
||||
self.tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast" = (
|
||||
AutoTokenizer.from_pretrained(
|
||||
model_name,
|
||||
tokenizer_name or model_name,
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
)
|
||||
|
||||
# don't put this import at the top level
|
||||
# it will call torch.accelerator.device_count()
|
||||
from transformers import AutoProcessor
|
||||
if processor is not None:
|
||||
self.processor = processor
|
||||
else:
|
||||
# don't put this import at the top level
|
||||
# it will call torch.accelerator.device_count()
|
||||
from transformers import AutoProcessor
|
||||
|
||||
self.processor = AutoProcessor.from_pretrained(
|
||||
model_name,
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
self.processor = AutoProcessor.from_pretrained(
|
||||
model_name,
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
if skip_tokenizer_init:
|
||||
if self.processor is None:
|
||||
raise ValueError(
|
||||
"skip_tokenizer_init=True requires processor initialization."
|
||||
)
|
||||
self.tokenizer = self.processor.tokenizer
|
||||
|
||||
def get_inputs(
|
||||
@@ -520,6 +533,12 @@ class HfRunner:
|
||||
all_inputs: list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]] = []
|
||||
for i, prompt in enumerate(prompts):
|
||||
if isinstance(prompt, str):
|
||||
if self.processor is None:
|
||||
raise RuntimeError(
|
||||
"HfRunner.processor is not initialized. "
|
||||
"Pass processor=... to HfRunner or set "
|
||||
"hf_model.processor before generation."
|
||||
)
|
||||
# Create a copy to avoid modifying the original dict
|
||||
processor_kwargs = (
|
||||
tokenization_kwargs.copy()
|
||||
@@ -617,6 +636,10 @@ class HfRunner:
|
||||
use_cache=True,
|
||||
**kwargs,
|
||||
)
|
||||
if self.processor is None:
|
||||
raise RuntimeError(
|
||||
"HfRunner.processor is not initialized; cannot decode output."
|
||||
)
|
||||
output_str = self.processor.batch_decode(
|
||||
output_ids,
|
||||
skip_special_tokens=True,
|
||||
|
||||
@@ -805,6 +805,32 @@ VLM_TEST_SETTINGS = {
|
||||
max_num_seqs=2,
|
||||
patch_hf_runner=model_utils.molmo_patch_hf_runner,
|
||||
),
|
||||
"moondream3": VLMTestInfo(
|
||||
models=["moondream/moondream3-preview"],
|
||||
test_type=VLMTestType.IMAGE,
|
||||
prompt_formatter=identity,
|
||||
img_idx_to_prompt=lambda idx: "<|endoftext|><image>",
|
||||
# Common-image coverage here targets query/caption. The native
|
||||
# detect/point skills are not exposed by vLLM.
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<vlm_image><|md_reserved_0|>query<|md_reserved_1|>What is this sign?<|md_reserved_2|>", # noqa: E501
|
||||
"cherry_blossom": (
|
||||
"<vlm_image><|md_reserved_0|>query<|md_reserved_1|>What season is shown?<|md_reserved_2|>" # noqa: E501
|
||||
),
|
||||
}
|
||||
),
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
dtype="bfloat16",
|
||||
hf_processor=model_utils.moondream3_processor,
|
||||
patch_hf_runner=model_utils.moondream3_patch_hf_runner,
|
||||
# Single size factor to avoid GPU OOM when running multiple test
|
||||
# cases sequentially (9B MoE model uses ~18 GiB per instance).
|
||||
image_size_factors=[(1.0,)],
|
||||
# Moondream3 is 9B params with MoE, needs significant GPU memory
|
||||
marks=[large_gpu_mark(min_gb=48)],
|
||||
),
|
||||
"ovis1_6-gemma2": VLMTestInfo(
|
||||
models=["AIDC-AI/Ovis1.6-Gemma2-9B"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Generation tests for Moondream3 query and caption support."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, ImageTestAssets
|
||||
from ....utils import large_gpu_mark, multi_gpu_test
|
||||
|
||||
MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
|
||||
MOONDREAM3_TOKENIZER = "moondream/starmie-v1"
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What color is the stop sign?<|md_reserved_2|>", # noqa: E501
|
||||
"cherry_blossom": "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What color are the flowers?<|md_reserved_2|>", # noqa: E501
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def make_query_prompt(question: str) -> str:
|
||||
"""Create a direct-answer query prompt for Moondream3."""
|
||||
return (
|
||||
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
|
||||
f"{question}<|md_reserved_2|>"
|
||||
)
|
||||
|
||||
|
||||
def make_caption_prompt(length: str = "normal") -> str:
|
||||
"""Create a caption prompt for Moondream3."""
|
||||
return (
|
||||
"<|endoftext|><image><|md_reserved_0|>"
|
||||
f"describe<|md_reserved_1|>{length}<|md_reserved_2|>"
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@large_gpu_mark(min_gb=80)
|
||||
def test_tensor_parallel(image_assets: ImageTestAssets):
|
||||
import gc
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed.parallel_state import destroy_model_parallel
|
||||
|
||||
destroy_model_parallel()
|
||||
gc.collect()
|
||||
current_platform.empty_cache()
|
||||
|
||||
llm = LLM(
|
||||
model=MOONDREAM3_MODEL_ID,
|
||||
tokenizer=MOONDREAM3_TOKENIZER,
|
||||
trust_remote_code=True,
|
||||
dtype="bfloat16",
|
||||
tensor_parallel_size=2,
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
gpu_memory_utilization=0.45,
|
||||
)
|
||||
|
||||
image = image_assets[0].pil_image
|
||||
prompt = make_query_prompt("What color is the stop sign?")
|
||||
|
||||
try:
|
||||
outputs = llm.generate(
|
||||
{"prompt": prompt, "multi_modal_data": {"image": image}},
|
||||
SamplingParams(max_tokens=20, temperature=0),
|
||||
)
|
||||
|
||||
assert len(outputs) > 0
|
||||
assert outputs[0].outputs[0].text is not None
|
||||
finally:
|
||||
del llm
|
||||
gc.collect()
|
||||
current_platform.empty_cache()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info("Moondream3ForCausalLM")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
from vllm import LLM
|
||||
|
||||
try:
|
||||
return LLM(
|
||||
model=MOONDREAM3_MODEL_ID,
|
||||
tokenizer=MOONDREAM3_TOKENIZER,
|
||||
trust_remote_code=True,
|
||||
dtype="bfloat16",
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
gpu_memory_utilization=0.45,
|
||||
)
|
||||
except Exception as exc:
|
||||
pytest.skip(f"Failed to load {MOONDREAM3_MODEL_ID}: {exc}")
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=48)
|
||||
def test_model_loading(llm):
|
||||
assert llm is not None
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=48)
|
||||
def test_query_skill(llm, image_assets: ImageTestAssets):
|
||||
from vllm import SamplingParams
|
||||
|
||||
image = image_assets[0].pil_image
|
||||
prompt = make_query_prompt("What color is the stop sign?")
|
||||
|
||||
outputs = llm.generate(
|
||||
{"prompt": prompt, "multi_modal_data": {"image": image}},
|
||||
SamplingParams(max_tokens=50, temperature=0),
|
||||
)
|
||||
|
||||
output_text = outputs[0].outputs[0].text
|
||||
assert output_text is not None
|
||||
assert len(output_text) > 0
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=48)
|
||||
def test_caption_skill(llm, image_assets: ImageTestAssets):
|
||||
from vllm import SamplingParams
|
||||
|
||||
image = image_assets[1].pil_image
|
||||
prompt = make_caption_prompt()
|
||||
|
||||
outputs = llm.generate(
|
||||
{"prompt": prompt, "multi_modal_data": {"image": image}},
|
||||
SamplingParams(max_tokens=100, temperature=0),
|
||||
)
|
||||
|
||||
output_text = outputs[0].outputs[0].text
|
||||
assert output_text is not None
|
||||
assert len(output_text) > 0
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=48)
|
||||
def test_batched_inference(llm, image_assets: ImageTestAssets):
|
||||
from vllm import SamplingParams
|
||||
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
prompts = [
|
||||
{"prompt": prompt, "multi_modal_data": {"image": img}}
|
||||
for img, prompt in zip(images, HF_IMAGE_PROMPTS)
|
||||
]
|
||||
|
||||
outputs = llm.generate(prompts, SamplingParams(max_tokens=50, temperature=0))
|
||||
|
||||
assert len(outputs) == len(images)
|
||||
for output in outputs:
|
||||
assert output.outputs[0].text is not None
|
||||
assert len(output.outputs[0].text) > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asset_name", ["stop_sign", "cherry_blossom"])
|
||||
@large_gpu_mark(min_gb=48)
|
||||
def test_image_assets(llm, image_assets: ImageTestAssets, asset_name: str):
|
||||
from vllm import SamplingParams
|
||||
|
||||
asset_idx = 0 if asset_name == "stop_sign" else 1
|
||||
image = image_assets[asset_idx].pil_image
|
||||
prompt = HF_IMAGE_PROMPTS[asset_idx]
|
||||
|
||||
outputs = llm.generate(
|
||||
{"prompt": prompt, "multi_modal_data": {"image": image}},
|
||||
SamplingParams(max_tokens=50, temperature=0),
|
||||
)
|
||||
|
||||
output_text = outputs[0].outputs[0].text
|
||||
assert output_text is not None
|
||||
assert len(output_text) > 0
|
||||
@@ -38,6 +38,7 @@ def run_test(
|
||||
limit_mm_per_prompt: dict[str, int],
|
||||
vllm_runner_kwargs: dict[str, Any] | None,
|
||||
hf_model_kwargs: dict[str, Any] | None,
|
||||
hf_processor: Callable[[str], Any] | None,
|
||||
patch_hf_runner: Callable[[HfRunner], HfRunner] | None,
|
||||
runner: RunnerOption = "auto",
|
||||
distributed_executor_backend: str | None = None,
|
||||
@@ -116,8 +117,18 @@ def run_test(
|
||||
)
|
||||
vllm_outputs_per_mm.append(vllm_output)
|
||||
|
||||
hf_runner_kwargs: dict[str, Any] = {}
|
||||
if model_info.tokenizer:
|
||||
hf_runner_kwargs["tokenizer_name"] = model_info.tokenizer
|
||||
if hf_processor is not None:
|
||||
hf_runner_kwargs["processor"] = hf_processor(model)
|
||||
|
||||
hf_model = hf_runner(
|
||||
model, dtype=dtype, auto_cls=auto_cls, model_kwargs=hf_model_kwargs
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=auto_cls,
|
||||
model_kwargs=hf_model_kwargs,
|
||||
**hf_runner_kwargs,
|
||||
)
|
||||
|
||||
# Some models need to patch things like the model processor, e.g., internvl
|
||||
|
||||
@@ -1336,3 +1336,221 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
|
||||
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign, assignment]
|
||||
hf_model.model.generate = patched_generate # type: ignore[method-assign]
|
||||
return hf_model
|
||||
|
||||
|
||||
def moondream3_processor(model: str):
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
return Moondream3Processor.from_pretrained(model, trust_remote_code=True)
|
||||
|
||||
|
||||
def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
"""Patch HfRunner for Moondream3."""
|
||||
moondream_processor = hf_model.processor
|
||||
|
||||
def processor(*args, text="", images=None, **kwargs):
|
||||
if images is None:
|
||||
return moondream_processor(text=text, **kwargs)
|
||||
|
||||
images_list = [images] if isinstance(images, Image) else images
|
||||
return moondream_processor(images=images_list, text=text, **kwargs)
|
||||
|
||||
hf_model.processor = processor
|
||||
|
||||
# Expose the LM head for logprob extraction.
|
||||
hf_model.model.get_output_embeddings = lambda: hf_model.model.model.text.lm_head
|
||||
|
||||
native_model = hf_model.model.model # MoondreamModel instance
|
||||
|
||||
from torch.nn import functional as F
|
||||
|
||||
from vllm.model_executor.models.moondream3 import reconstruct_from_crops
|
||||
|
||||
# Resolve the placeholder tokens from the tokenizer instead of hard-coding.
|
||||
image_placeholder_ids = moondream_processor.tokenizer.encode(
|
||||
"<image>", add_special_tokens=False
|
||||
)
|
||||
|
||||
def _normalize_tiling(tilings):
|
||||
"""Extract (h, w) tuple from various tiling container formats."""
|
||||
tiling = tilings
|
||||
if isinstance(tiling, torch.Tensor):
|
||||
tiling = tuple(tiling.squeeze().tolist())
|
||||
elif isinstance(tiling, (list, tuple)):
|
||||
t0 = tiling[0]
|
||||
if isinstance(t0, torch.Tensor):
|
||||
tiling = tuple(t0.tolist())
|
||||
elif isinstance(t0, (list, tuple)):
|
||||
tiling = tuple(t0)
|
||||
return tiling
|
||||
|
||||
def _encode_vision(pixel_values, tilings):
|
||||
"""Run preprocessed crops through vision encoder + projection."""
|
||||
device = native_model.device
|
||||
dtype = native_model.vision.pos_emb.dtype
|
||||
config = native_model.config
|
||||
|
||||
pv = pixel_values
|
||||
while pv.dim() > 4:
|
||||
pv = pv.squeeze(0)
|
||||
pv = pv.to(device=device, dtype=dtype)
|
||||
|
||||
features = native_model._vis_enc(pv)
|
||||
grid_size = config.vision.crop_size // config.vision.enc_patch_size
|
||||
global_feat = features[0]
|
||||
|
||||
if features.shape[0] > 1 and tilings is not None:
|
||||
tiling = _normalize_tiling(tilings)
|
||||
local = features[1:].view(-1, grid_size, grid_size, config.vision.enc_dim)
|
||||
reconstructed = reconstruct_from_crops(
|
||||
local,
|
||||
tiling,
|
||||
config.vision.overlap_margin,
|
||||
patch_size=1,
|
||||
)
|
||||
else:
|
||||
reconstructed = global_feat.view(
|
||||
grid_size, grid_size, config.vision.enc_dim
|
||||
)
|
||||
|
||||
return native_model._vis_proj(global_feat, reconstructed)
|
||||
|
||||
def _find_subsequence(seq, subseq):
|
||||
"""Find start index of subseq in seq, or None."""
|
||||
n = len(subseq)
|
||||
for i in range(len(seq) - n + 1):
|
||||
if seq[i : i + n] == subseq:
|
||||
return i
|
||||
return None
|
||||
|
||||
def _generate(
|
||||
self,
|
||||
input_ids=None,
|
||||
pixel_values=None,
|
||||
tilings=None,
|
||||
attention_mask=None,
|
||||
**kwargs,
|
||||
):
|
||||
max_new_tokens = kwargs.get("max_new_tokens", 128)
|
||||
return_dict = kwargs.get("return_dict_in_generate", False)
|
||||
output_hs = kwargs.get("output_hidden_states", False)
|
||||
|
||||
if pixel_values is None:
|
||||
sequences = input_ids
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=() if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
# Processor may return lists; extract the single element.
|
||||
if isinstance(pixel_values, (list, tuple)):
|
||||
pixel_values = pixel_values[0]
|
||||
if (
|
||||
isinstance(tilings, (list, tuple))
|
||||
and tilings
|
||||
and not isinstance(tilings[0], int)
|
||||
):
|
||||
tilings = tilings[0]
|
||||
|
||||
hf_model.model._setup_caches()
|
||||
native_model.use_flex_decoding = False
|
||||
|
||||
device = native_model.device
|
||||
config = native_model.config
|
||||
|
||||
with torch.inference_mode():
|
||||
for block in native_model.text.blocks:
|
||||
block.kv_cache.k_cache.zero_()
|
||||
block.kv_cache.v_cache.zero_()
|
||||
|
||||
img_emb = _encode_vision(pixel_values, tilings)
|
||||
|
||||
bos_emb = F.embedding(
|
||||
torch.tensor([[config.tokenizer.bos_id]], device=device),
|
||||
native_model.text.wte,
|
||||
)
|
||||
img_input = torch.cat([bos_emb, img_emb.unsqueeze(0)], dim=1)
|
||||
prefix_len = img_input.size(1)
|
||||
|
||||
mask = native_model.attn_mask[:, :, :prefix_len, :]
|
||||
pos_ids = torch.arange(prefix_len, dtype=torch.long, device=device)
|
||||
native_model._prefill(img_input, mask, pos_ids, None)
|
||||
|
||||
ids = input_ids.squeeze(0).tolist()
|
||||
img_start = _find_subsequence(ids, image_placeholder_ids)
|
||||
|
||||
if img_start is None:
|
||||
sequences = input_ids
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=() if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
prompt_tokens = ids[img_start + len(image_placeholder_ids) :]
|
||||
|
||||
if not prompt_tokens:
|
||||
sequences = input_ids
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=() if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
prompt_tensor = torch.tensor([prompt_tokens], device=device)
|
||||
prompt_emb = F.embedding(prompt_tensor, native_model.text.wte)
|
||||
prompt_len = prompt_emb.size(1)
|
||||
|
||||
mask = native_model.attn_mask[:, :, prefix_len : prefix_len + prompt_len, :]
|
||||
pos_ids = torch.arange(
|
||||
prefix_len,
|
||||
prefix_len + prompt_len,
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
)
|
||||
hidden = native_model._prefill(prompt_emb, mask, pos_ids, None)
|
||||
pos = prefix_len + prompt_len
|
||||
|
||||
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
|
||||
logits = native_model.text.lm_head(hidden_last.squeeze(1))
|
||||
|
||||
generated = []
|
||||
all_hidden_states = []
|
||||
# Record the hidden state that predicted each generated token.
|
||||
prev_hs = hidden_last
|
||||
for _ in range(max_new_tokens):
|
||||
next_token = logits.argmax(dim=-1).item()
|
||||
if next_token == 0:
|
||||
break
|
||||
generated.append(next_token)
|
||||
if output_hs:
|
||||
all_hidden_states.append((prev_hs,))
|
||||
|
||||
next_emb = F.embedding(
|
||||
torch.tensor([[next_token]], device=device),
|
||||
native_model.text.wte,
|
||||
)
|
||||
mask = native_model.attn_mask[:, :, pos : pos + 1, :]
|
||||
pos_ids_step = torch.tensor([pos], dtype=torch.long, device=device)
|
||||
hidden = native_model._prefill(next_emb, mask, pos_ids_step, None)
|
||||
hidden_last = native_model.text.post_ln(hidden[:, -1:, :])
|
||||
prev_hs = hidden_last
|
||||
logits = native_model.text.lm_head(hidden_last.squeeze(1))
|
||||
pos += 1
|
||||
|
||||
result_ids = ids + generated
|
||||
sequences = torch.tensor([result_ids], device=device)
|
||||
|
||||
if return_dict:
|
||||
return types.SimpleNamespace(
|
||||
sequences=sequences,
|
||||
hidden_states=tuple(all_hidden_states) if output_hs else None,
|
||||
)
|
||||
return sequences
|
||||
|
||||
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
|
||||
return hf_model
|
||||
|
||||
@@ -133,6 +133,7 @@ class VLMTestInfo(NamedTuple):
|
||||
|
||||
# Exposed options for HF runner
|
||||
hf_model_kwargs: dict[str, Any] | None = None
|
||||
hf_processor: Callable[[str], Any] | None = None
|
||||
# Indicates we should explicitly pass the EOS from the tokenizer
|
||||
use_tokenizer_eos: bool = False
|
||||
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM
|
||||
@@ -196,6 +197,7 @@ class VLMTestInfo(NamedTuple):
|
||||
"comparator": self.comparator,
|
||||
"get_stop_token_ids": self.get_stop_token_ids,
|
||||
"hf_model_kwargs": self.hf_model_kwargs,
|
||||
"hf_processor": self.hf_processor,
|
||||
"stop_str": self.stop_str,
|
||||
"patch_hf_runner": self.patch_hf_runner,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,553 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Moondream3 multimodal processing.
|
||||
|
||||
Includes:
|
||||
- Processor creation and application tests
|
||||
- Image tokenization and placeholder expansion tests
|
||||
- Tiling and cropping logic tests (CPU-based)
|
||||
- Pixel normalization tests
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
|
||||
# Expected multimodal prefix: BOS + 729 image tokens.
|
||||
EXPECTED_IMAGE_TOKENS = 730
|
||||
# Vision encoder constants
|
||||
CROP_SIZE = 378
|
||||
PATCH_SIZE = 14
|
||||
MAX_CROPS = 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_creation(model_id: str):
|
||||
"""Test that Moondream3 processor can be created."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
assert processor is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_apply(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that Moondream3 processor can process inputs.
|
||||
|
||||
NOTE: The prompt includes the leading BOS token because Moondream3
|
||||
pre-fills BOS and image embeddings together.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert "prompt_token_ids" in processed_inputs
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_pixel_values(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that pixel values are correctly produced."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
# Check mm_kwargs contains pixel_values
|
||||
mm_kwargs = processed_inputs.get("mm_kwargs")
|
||||
assert mm_kwargs is not None
|
||||
mm_data_result = mm_kwargs.get_data()
|
||||
assert "pixel_values" in mm_data_result
|
||||
|
||||
# Verify pixel_values shape
|
||||
pixel_values = mm_data_result["pixel_values"]
|
||||
assert pixel_values.dim() == 5 # [batch, num_crops, C, H, W]
|
||||
assert pixel_values.shape[2] == 3 # RGB channels
|
||||
assert pixel_values.shape[3] == 378 # crop height
|
||||
assert pixel_values.shape[4] == 378 # crop width
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_image_token_expansion(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that <image> placeholder is expanded to correct number of tokens."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe.<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_multi_crop_tiling(
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that large images produce correct multi-crop tiling."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
# Create a large image that requires multiple crops
|
||||
large_image = Image.new("RGB", (1000, 1000), color="blue")
|
||||
pixel_values, tiling = processor.preprocess_image(large_image)
|
||||
|
||||
# Large images should produce more than 1x1 tiling
|
||||
assert tiling[0] >= 1 and tiling[1] >= 1
|
||||
# Check that we have global crop + local crops
|
||||
expected_crops = tiling[0] * tiling[1] + 1
|
||||
assert pixel_values.shape[0] == expected_crops
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_size",
|
||||
[
|
||||
(500, 500),
|
||||
(800, 600),
|
||||
(1920, 1080),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_tiling_various_sizes(
|
||||
image_size: tuple[int, int],
|
||||
model_id: str,
|
||||
):
|
||||
"""Test tiling with various image sizes."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
width, height = image_size
|
||||
image = Image.new("RGB", (width, height), color="red")
|
||||
pixel_values, tiling = processor.preprocess_image(image)
|
||||
|
||||
# Basic shape checks
|
||||
assert pixel_values.dim() == 4 # [num_crops, C, H, W]
|
||||
assert pixel_values.shape[1] == 3 # RGB
|
||||
assert pixel_values.shape[2] == 378 # crop height
|
||||
assert pixel_values.shape[3] == 378 # crop width
|
||||
|
||||
# Tiling should respect max_crops (12)
|
||||
assert tiling[0] * tiling[1] <= 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_pixel_normalization(
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that pixel values are normalized to [-1, 1] range."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
# Create test image
|
||||
image = Image.new("RGB", (378, 378), color="green")
|
||||
pixel_values, _ = processor.preprocess_image(image)
|
||||
|
||||
# Normalization: (x - 0.5) / 0.5 = 2*x - 1
|
||||
# For input [0, 1], output should be [-1, 1]
|
||||
assert pixel_values.min() >= -1.0
|
||||
assert pixel_values.max() <= 1.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_chat_template_with_image(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that chat template correctly formats BOS + image + prompt."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = ctx.tokenizer
|
||||
|
||||
# Use the chat template format
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
token_ids = processed_inputs["prompt_token_ids"]
|
||||
|
||||
# BOS token (<|endoftext|>) should be token ID 0
|
||||
bos_token_id = tokenizer.encode("<|endoftext|>", add_special_tokens=False)[0]
|
||||
assert bos_token_id == 0
|
||||
|
||||
# First token should be BOS
|
||||
assert token_ids[0] == bos_token_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
pytest.param(
|
||||
[
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.invalid/image.png"},
|
||||
},
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
],
|
||||
id="image-first",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.invalid/image.png"},
|
||||
},
|
||||
],
|
||||
id="text-first",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_chat_template_content_list_uses_moondream_image_prefix(
|
||||
image_assets: ImageTestAssets,
|
||||
content: list[dict[str, object]],
|
||||
model_id: str,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
|
||||
prompt = hf_processor.tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": content}],
|
||||
chat_template=hf_processor.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
|
||||
expected_prompt = (
|
||||
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
|
||||
"What is in this image?<|md_reserved_2|>"
|
||||
)
|
||||
assert prompt == expected_prompt
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"image": [image_assets[0].pil_image]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_bos_token_always_first(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that BOS token (ID 0) is always at position 0."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Start with BOS token explicitly
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe this image.<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
token_ids = processed_inputs["prompt_token_ids"]
|
||||
|
||||
# Token ID 0 (<|endoftext|>) should be the first token
|
||||
assert token_ids[0] == 0, (
|
||||
f"Expected BOS token (0) at position 0, got {token_ids[0]}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_with_small_image(
|
||||
model_id: str,
|
||||
):
|
||||
"""Test processor with image smaller than crop size."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
# Small image (smaller than crop size)
|
||||
small_image = Image.new("RGB", (100, 100), color="yellow")
|
||||
pixel_values, tiling = processor.preprocess_image(small_image)
|
||||
|
||||
# Small images should use 1x1 tiling
|
||||
assert tiling == (1, 1)
|
||||
# Should have 2 crops (global + 1 local)
|
||||
assert pixel_values.shape[0] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_kind",
|
||||
[
|
||||
pytest.param("numpy_hwc", id="numpy-hwc"),
|
||||
pytest.param("numpy_chw", id="numpy-chw"),
|
||||
pytest.param("torch_chw", id="torch-chw"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_preprocess_image_accepts_non_pil_inputs(
|
||||
image_assets: ImageTestAssets,
|
||||
image_kind: str,
|
||||
model_id: str,
|
||||
):
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
pil_image = image_assets[0].pil_image.convert("RGB")
|
||||
hwc_array = np.asarray(pil_image)
|
||||
expected_pixel_values, expected_tiling = processor.preprocess_image(pil_image)
|
||||
|
||||
if image_kind == "numpy_hwc":
|
||||
image = hwc_array
|
||||
elif image_kind == "numpy_chw":
|
||||
image = np.transpose(hwc_array, (2, 0, 1))
|
||||
else:
|
||||
image = torch.from_numpy(np.transpose(hwc_array, (2, 0, 1)).copy())
|
||||
|
||||
pixel_values, tiling = processor.preprocess_image(image)
|
||||
|
||||
assert tiling == expected_tiling
|
||||
assert pixel_values.shape == expected_pixel_values.shape
|
||||
assert pixel_values.dtype == torch.bfloat16
|
||||
assert torch.equal(pixel_values, expected_pixel_values)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_kind", ["numpy_chw", "torch_chw"])
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_apply_accepts_non_pil_image_inputs(
|
||||
image_assets: ImageTestAssets,
|
||||
image_kind: str,
|
||||
model_id: str,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
hwc_array = np.asarray(image_assets[0].pil_image.convert("RGB"))
|
||||
chw_array = np.transpose(hwc_array, (2, 0, 1)).copy()
|
||||
image = chw_array if image_kind == "numpy_chw" else torch.from_numpy(chw_array)
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"image": [image]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
mm_kwargs = processed_inputs["mm_kwargs"].get_data()
|
||||
assert mm_kwargs["pixel_values"].shape[2:] == (3, 378, 378)
|
||||
|
||||
|
||||
class TestMoondream3TilingLogic:
|
||||
"""CPU-based tests for Moondream3 tiling selection logic.
|
||||
|
||||
These tests validate the select_tiling() function which determines
|
||||
how images are divided into crops for the vision encoder.
|
||||
"""
|
||||
|
||||
def test_small_image_no_tiling(self):
|
||||
"""Small images should use 1x1 tiling."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=300, width=300, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
assert tiling == (1, 1)
|
||||
|
||||
def test_exact_crop_size(self):
|
||||
"""Image exactly at crop size should use 1x1."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=CROP_SIZE, width=CROP_SIZE, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
assert tiling == (1, 1)
|
||||
|
||||
def test_large_square_image(self):
|
||||
"""Large square image should use multiple tiles."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=800, width=800, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert h_tiles >= 2
|
||||
assert w_tiles >= 2
|
||||
assert h_tiles * w_tiles <= MAX_CROPS
|
||||
|
||||
def test_wide_image(self):
|
||||
"""Wide image should have more width tiles."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=400, width=1200, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert w_tiles >= h_tiles
|
||||
|
||||
def test_tall_image(self):
|
||||
"""Tall image should have more height tiles."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=1200, width=400, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert h_tiles >= w_tiles
|
||||
|
||||
def test_respects_max_crops(self):
|
||||
"""Tiling should not exceed max_crops."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=2000, width=2000, crop_size=CROP_SIZE, max_crops=4
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert h_tiles * w_tiles <= 4
|
||||
|
||||
|
||||
class TestMoondream3VisionShapes:
|
||||
"""CPU-based tests for vision encoder expected shapes.
|
||||
|
||||
These tests verify the mathematical relationships between
|
||||
crop size, patch size, and token counts.
|
||||
"""
|
||||
|
||||
def test_expected_patch_count(self):
|
||||
"""Test 378/14 = 27 patches per side, 729 total."""
|
||||
patches_per_side = CROP_SIZE // PATCH_SIZE
|
||||
total_patches = patches_per_side**2
|
||||
|
||||
assert patches_per_side == 27
|
||||
assert total_patches == EXPECTED_IMAGE_TOKENS - 1
|
||||
|
||||
def test_patch_embedding_input_dim(self):
|
||||
"""Test patch embedding input dimension."""
|
||||
channels = 3
|
||||
input_dim = PATCH_SIZE * PATCH_SIZE * channels
|
||||
|
||||
assert input_dim == 14 * 14 * 3
|
||||
assert input_dim == 588
|
||||
|
||||
|
||||
class TestMoondream3TauAttention:
|
||||
"""CPU-based tests for tau attention scaling components.
|
||||
|
||||
These tests validate the tau attention formula used in Moondream3:
|
||||
- Token-based: tok_q = tanh(gelu(qkv) @ tau_wq.T)
|
||||
- Position-based: tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5)
|
||||
"""
|
||||
|
||||
def test_tau_position_range(self):
|
||||
"""Test tau position scaling produces values in valid range."""
|
||||
num_heads = 32
|
||||
seq_len = 100
|
||||
|
||||
tau_alpha = torch.randn(num_heads)
|
||||
positions = torch.arange(seq_len)
|
||||
|
||||
pos_float = (positions.float() + 1.0).clamp(min=1e-6)
|
||||
pos_log = pos_float.log()
|
||||
tau_pos = 1.0 + (torch.sigmoid(tau_alpha[:, None] * pos_log[None, :]) - 0.5)
|
||||
|
||||
assert tau_pos.shape == (num_heads, seq_len)
|
||||
# tau_pos should be between 0.5 and 1.5
|
||||
assert tau_pos.min() >= 0.5
|
||||
assert tau_pos.max() <= 1.5
|
||||
|
||||
def test_tau_token_output_range(self):
|
||||
"""Test tau token scaling output is bounded by tanh."""
|
||||
import torch.nn.functional as F
|
||||
|
||||
seq_len = 100
|
||||
qkv_dim = 6144 # 2048 * 3
|
||||
num_heads = 32
|
||||
|
||||
qkv = torch.randn(seq_len, qkv_dim)
|
||||
tau_wq = torch.randn(num_heads, qkv_dim)
|
||||
|
||||
tok_feat = F.gelu(qkv)
|
||||
tok_q = torch.tanh(tok_feat @ tau_wq.t())
|
||||
|
||||
assert tok_q.shape == (seq_len, num_heads)
|
||||
# tanh output is bounded by [-1, 1]
|
||||
assert tok_q.min() >= -1.0
|
||||
assert tok_q.max() <= 1.0
|
||||
@@ -1122,6 +1122,16 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
extras={"olmo": "allenai/Molmo-7B-O-0924"},
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"Moondream3ForCausalLM": _HfExamplesInfo(
|
||||
"moondream/moondream3-preview",
|
||||
tokenizer="moondream/starmie-v1",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"HfMoondream": _HfExamplesInfo(
|
||||
"moondream/moondream3-preview",
|
||||
tokenizer="moondream/starmie-v1",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"Molmo2ForConditionalGeneration": _HfExamplesInfo(
|
||||
"allenai/Molmo2-8B",
|
||||
extras={"olmo": "allenai/Molmo2-O-7B"},
|
||||
|
||||
@@ -523,6 +523,17 @@ def dummy_hf_overrides(
|
||||
|
||||
text_config.update(update_dict)
|
||||
|
||||
# Update n_layers and moe configs for Moondream3 model
|
||||
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
|
||||
text_config.update(
|
||||
{
|
||||
"n_layers": num_hidden_layers,
|
||||
"moe_num_experts": num_experts,
|
||||
"moe_experts_per_token": 2,
|
||||
"moe_start_layer": num_hidden_layers,
|
||||
}
|
||||
)
|
||||
|
||||
if hasattr(hf_config, "vision_config"):
|
||||
hf_config.vision_config.update(
|
||||
{
|
||||
@@ -531,6 +542,9 @@ def dummy_hf_overrides(
|
||||
}
|
||||
)
|
||||
|
||||
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
|
||||
hf_config.vision_config.update({"enc_n_layers": 1})
|
||||
|
||||
# e.g.: ibm-granite/granite-speech-3.3-2b
|
||||
if hasattr(hf_config, "encoder_config"):
|
||||
hf_config.encoder_config.update(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -484,6 +484,8 @@ _MULTIMODAL_MODELS = {
|
||||
),
|
||||
"MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
|
||||
"Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
|
||||
"Moondream3ForCausalLM": ("moondream3", "Moondream3ForCausalLM"),
|
||||
"HfMoondream": ("moondream3", "Moondream3ForCausalLM"),
|
||||
"NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
|
||||
"NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
|
||||
"NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
|
||||
|
||||
@@ -114,6 +114,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
mlp_speculator="MLPSpeculatorConfig",
|
||||
medusa="MedusaConfig",
|
||||
midashenglm="MiDashengLMConfig",
|
||||
moondream3="Moondream3Config",
|
||||
eagle="EAGLEConfig",
|
||||
speculators="SpeculatorsConfig",
|
||||
nemotron="NemotronConfig",
|
||||
|
||||
@@ -50,6 +50,9 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"MedusaConfig": "vllm.transformers_utils.configs.medusa",
|
||||
"MiDashengLMConfig": "vllm.transformers_utils.configs.midashenglm",
|
||||
"MLPSpeculatorConfig": "vllm.transformers_utils.configs.mlp_speculator",
|
||||
"Moondream3Config": "vllm.transformers_utils.configs.moondream3",
|
||||
"Moondream3TextConfig": "vllm.transformers_utils.configs.moondream3",
|
||||
"Moondream3VisionConfig": "vllm.transformers_utils.configs.moondream3",
|
||||
"MoonViTConfig": "vllm.transformers_utils.configs.moonvit",
|
||||
"KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear",
|
||||
"KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl",
|
||||
@@ -111,6 +114,9 @@ __all__ = [
|
||||
"MedusaConfig",
|
||||
"MiDashengLMConfig",
|
||||
"MLPSpeculatorConfig",
|
||||
"Moondream3Config",
|
||||
"Moondream3TextConfig",
|
||||
"Moondream3VisionConfig",
|
||||
"MoonViTConfig",
|
||||
"KimiLinearConfig",
|
||||
"KimiVLConfig",
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Configuration for Moondream3 model."""
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
|
||||
class Moondream3VisionConfig(PretrainedConfig):
|
||||
"""Vision encoder configuration for Moondream3."""
|
||||
|
||||
model_type = "moondream3_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
enc_dim: int = 1152,
|
||||
enc_patch_size: int = 14,
|
||||
enc_n_layers: int = 27,
|
||||
enc_ff_dim: int = 4304,
|
||||
enc_n_heads: int = 16,
|
||||
proj_inner_dim: int = 8192,
|
||||
crop_size: int = 378,
|
||||
max_crops: int = 12,
|
||||
overlap_margin: int = 4,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.enc_dim = enc_dim
|
||||
self.enc_patch_size = enc_patch_size
|
||||
self.enc_n_layers = enc_n_layers
|
||||
self.enc_ff_dim = enc_ff_dim
|
||||
self.enc_n_heads = enc_n_heads
|
||||
self.proj_inner_dim = proj_inner_dim
|
||||
self.crop_size = crop_size
|
||||
self.max_crops = max_crops
|
||||
self.overlap_margin = overlap_margin
|
||||
|
||||
# Standard HuggingFace attributes for vision config
|
||||
self.hidden_size = enc_dim
|
||||
self.num_attention_heads = enc_n_heads
|
||||
self.num_hidden_layers = enc_n_layers
|
||||
self.intermediate_size = enc_ff_dim
|
||||
self.patch_size = enc_patch_size
|
||||
self.image_size = crop_size
|
||||
|
||||
|
||||
class Moondream3TextConfig(PretrainedConfig):
|
||||
"""Text decoder configuration for Moondream3."""
|
||||
|
||||
model_type = "moondream3_text"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int = 2048,
|
||||
ff_dim: int = 8192,
|
||||
n_layers: int = 24,
|
||||
vocab_size: int = 51200,
|
||||
max_context: int = 4096,
|
||||
n_heads: int = 32,
|
||||
n_kv_heads: int = 32,
|
||||
prefix_attn: int = 730,
|
||||
rope_theta: float = 1500000.0,
|
||||
moe: dict | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# Store original moondream3 config names
|
||||
self.dim = dim
|
||||
self.ff_dim = ff_dim
|
||||
self.n_layers = n_layers
|
||||
self.n_heads = n_heads
|
||||
self.n_kv_heads = n_kv_heads
|
||||
self.prefix_attn = prefix_attn
|
||||
self.max_context = max_context
|
||||
self.rope_theta = rope_theta
|
||||
|
||||
# MoE config
|
||||
moe = moe or {}
|
||||
self.moe_start_layer = moe.get("start_layer", 4)
|
||||
self.moe_num_experts = moe.get("n_experts", 64)
|
||||
self.moe_experts_per_token = moe.get("n_experts_per_tok", 8)
|
||||
self.moe_expert_inner_dim = moe.get("expert_inner_dim", 1024)
|
||||
|
||||
# Standard HuggingFace attributes (required by vLLM)
|
||||
self.hidden_size = dim
|
||||
self.num_attention_heads = n_heads
|
||||
self.num_key_value_heads = n_kv_heads
|
||||
self.num_hidden_layers = n_layers
|
||||
self.intermediate_size = ff_dim
|
||||
self.vocab_size = vocab_size
|
||||
self.max_position_embeddings = max_context
|
||||
|
||||
# Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS.
|
||||
# Token 3 (<|md_reserved_2|>) is an answer delimiter that the model
|
||||
# implementation suppresses during generation.
|
||||
self.bos_token_id = 0
|
||||
self.eos_token_id = 0
|
||||
|
||||
# MoE standard attributes
|
||||
self.num_local_experts = self.moe_num_experts
|
||||
self.num_experts_per_tok = self.moe_experts_per_token
|
||||
|
||||
|
||||
class Moondream3Config(PretrainedConfig):
|
||||
"""Combined configuration for Moondream3 multimodal model."""
|
||||
|
||||
model_type = "moondream3"
|
||||
is_composition = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: dict | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
config = config or {}
|
||||
|
||||
# Parse text config
|
||||
text_config = config.get("text", {})
|
||||
self.text_config: Moondream3TextConfig = Moondream3TextConfig(**text_config)
|
||||
|
||||
# Parse vision config
|
||||
vision_config = config.get("vision", {})
|
||||
self.vision_config = Moondream3VisionConfig(**vision_config)
|
||||
|
||||
# Store the original config dict for model access
|
||||
self.config = config
|
||||
tokenizer_config = config.get("tokenizer", {})
|
||||
self.answer_token_id = tokenizer_config.get("answer_id", 3)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# Expose key attributes at top level for vLLM compatibility
|
||||
self.hidden_size = self.text_config.hidden_size
|
||||
self.num_attention_heads = self.text_config.num_attention_heads
|
||||
self.num_key_value_heads = self.text_config.num_key_value_heads
|
||||
self.num_hidden_layers = self.text_config.num_hidden_layers
|
||||
self.vocab_size = self.text_config.vocab_size
|
||||
self.intermediate_size = self.text_config.intermediate_size
|
||||
|
||||
# Moondream3 uses token 0 (<|endoftext|>) as both BOS and EOS.
|
||||
# Token 3 (<|md_reserved_2|>) is an answer delimiter that the model
|
||||
# implementation suppresses during generation.
|
||||
self.bos_token_id = 0
|
||||
self.eos_token_id = 0
|
||||
|
||||
def get_text_config(self, decoder: bool = False) -> "Moondream3TextConfig":
|
||||
"""Return the text config for vLLM's text_config detection.
|
||||
|
||||
Args:
|
||||
decoder: Ignored. Only used for encoder-decoder models.
|
||||
"""
|
||||
return self.text_config
|
||||
@@ -267,6 +267,7 @@ class ModelArchConfigConvertorBase:
|
||||
"bagel",
|
||||
"gemma3",
|
||||
"molmo2",
|
||||
"moondream3",
|
||||
"paligemma",
|
||||
"umm",
|
||||
)
|
||||
|
||||
@@ -23,6 +23,7 @@ __all__ = [
|
||||
"H2OVLProcessor",
|
||||
"HunYuanVLProcessor",
|
||||
"HunYuanVLImageProcessor",
|
||||
"Moondream3Processor",
|
||||
"InternVLProcessor",
|
||||
"IsaacProcessor",
|
||||
"KimiAudioProcessor",
|
||||
@@ -61,6 +62,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"MiMoOmniProcessor": "vllm.transformers_utils.processors.mimo_v2_omni",
|
||||
"MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
|
||||
"MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
|
||||
"Moondream3Processor": "vllm.transformers_utils.processors.moondream3",
|
||||
"NanoNemotronVLProcessor": "vllm.transformers_utils.processors.nano_nemotron_vl",
|
||||
"NemotronVLProcessor": "vllm.transformers_utils.processors.nemotron_vl",
|
||||
"LlamaNemotronVLEmbedProcessor": "vllm.transformers_utils.processors.nemotron_vl",
|
||||
|
||||
@@ -0,0 +1,541 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Custom processor for Moondream3 model."""
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoProcessor, BatchFeature
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
|
||||
from transformers.tokenization_utils_base import (
|
||||
PreTokenizedInput,
|
||||
PreTrainedTokenizerBase,
|
||||
TextInput,
|
||||
)
|
||||
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
__all__ = ["Moondream3Processor"]
|
||||
|
||||
|
||||
class Moondream3ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
|
||||
_defaults = {
|
||||
"text_kwargs": {
|
||||
"padding": False,
|
||||
},
|
||||
"images_kwargs": {
|
||||
"max_crops": 12,
|
||||
"overlap_margin": 4,
|
||||
"crop_size": 378,
|
||||
"patch_size": 14,
|
||||
"convert_to_rgb": True,
|
||||
"return_tensors": "pt",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def select_tiling(
|
||||
height: int, width: int, crop_size: int, max_crops: int
|
||||
) -> tuple[int, int]:
|
||||
"""Determine the optimal number of tiles to cover an image."""
|
||||
if height <= crop_size or width <= crop_size:
|
||||
return (1, 1)
|
||||
|
||||
min_h = math.ceil(height / crop_size)
|
||||
min_w = math.ceil(width / crop_size)
|
||||
|
||||
if min_h * min_w > max_crops:
|
||||
ratio = math.sqrt(max_crops / (min_h * min_w))
|
||||
return (max(1, math.floor(min_h * ratio)), max(1, math.floor(min_w * ratio)))
|
||||
|
||||
h_tiles = math.floor(math.sqrt(max_crops * height / width))
|
||||
w_tiles = math.floor(math.sqrt(max_crops * width / height))
|
||||
|
||||
h_tiles = max(h_tiles, min_h)
|
||||
w_tiles = max(w_tiles, min_w)
|
||||
|
||||
if h_tiles * w_tiles > max_crops:
|
||||
if w_tiles > h_tiles:
|
||||
w_tiles = math.floor(max_crops / h_tiles)
|
||||
else:
|
||||
h_tiles = math.floor(max_crops / w_tiles)
|
||||
|
||||
return (max(1, h_tiles), max(1, w_tiles))
|
||||
|
||||
|
||||
class Moondream3Processor(ProcessorMixin):
|
||||
"""
|
||||
Constructs a Moondream3 processor which handles image preprocessing
|
||||
and tokenization for the Moondream3 multimodal model.
|
||||
|
||||
Args:
|
||||
tokenizer: The tokenizer to use for text processing.
|
||||
chat_template: Optional chat template string.
|
||||
crop_size: Size of each image crop.
|
||||
max_crops: Maximum number of crops per image.
|
||||
overlap_margin: Margin for overlapping crops in patches.
|
||||
patch_size: Size of each patch.
|
||||
"""
|
||||
|
||||
attributes = ["tokenizer"]
|
||||
valid_kwargs = [
|
||||
"chat_template",
|
||||
"crop_size",
|
||||
"max_crops",
|
||||
"overlap_margin",
|
||||
"patch_size",
|
||||
]
|
||||
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
# Use separate tokenizer repo
|
||||
_tokenizer_repo = "moondream/starmie-v1"
|
||||
|
||||
# Default chat template for Moondream3
|
||||
# Moondream uses special tokens for prompting:
|
||||
# - Token 0 (<|endoftext|>): BOS token (ALWAYS present at position 0)
|
||||
# - Token 1 (<|md_reserved_0|>): Start of instruction
|
||||
# - Token 2 (<|md_reserved_1|>): Separator before question
|
||||
# - Token 3 (<|md_reserved_2|>): End of question / start of answer
|
||||
#
|
||||
# Task routing based on text prefix:
|
||||
# "caption [short|normal|long]" → describe<|md_reserved_1|>{length}
|
||||
# "describe [short|normal|long]" → describe<|md_reserved_1|>{length}
|
||||
# otherwise → query<|md_reserved_1|><text>
|
||||
#
|
||||
# Format with image:
|
||||
# <|endoftext|><image><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|>
|
||||
# Format without image:
|
||||
# <|endoftext|><|md_reserved_0|>{task}<|md_reserved_1|>{q}<|md_reserved_2|>
|
||||
_default_chat_template = (
|
||||
"{% for message in messages %}"
|
||||
"{% if message['role'] == 'user' %}"
|
||||
"{% if message['content'] is string %}"
|
||||
# Simple string content (with image assumed) - route by prefix
|
||||
"<|endoftext|><image><|md_reserved_0|>"
|
||||
"{% if message['content'] == 'caption' %}"
|
||||
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
|
||||
"{% elif message['content'].startswith('caption ') %}"
|
||||
"describe<|md_reserved_1|>{{ message['content'][8:] }}<|md_reserved_2|>"
|
||||
"{% elif message['content'] == 'describe' %}"
|
||||
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
|
||||
"{% elif message['content'].startswith('describe ') %}"
|
||||
"describe<|md_reserved_1|>{{ message['content'][9:] }}<|md_reserved_2|>"
|
||||
"{% else %}"
|
||||
"query<|md_reserved_1|>{{ message['content'] }}<|md_reserved_2|>"
|
||||
"{% endif %}"
|
||||
"{% else %}"
|
||||
# List content - build Moondream's image prefix independently of
|
||||
# OpenAI-style content part order, then render the text task.
|
||||
"<|endoftext|>"
|
||||
"{% for content in message['content'] %}"
|
||||
"{% if content['type'] in ['image', 'image_url', 'input_image', 'image_pil'] %}" # noqa: E501
|
||||
"<image>"
|
||||
"{% endif %}"
|
||||
"{% endfor %}"
|
||||
"{% for content in message['content'] %}"
|
||||
"{% if content['type'] == 'text' %}"
|
||||
"<|md_reserved_0|>"
|
||||
"{% if content['text'] == 'caption' %}"
|
||||
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
|
||||
"{% elif content['text'].startswith('caption ') %}"
|
||||
"describe<|md_reserved_1|>{{ content['text'][8:] }}<|md_reserved_2|>"
|
||||
"{% elif content['text'] == 'describe' %}"
|
||||
"describe<|md_reserved_1|>normal<|md_reserved_2|>"
|
||||
"{% elif content['text'].startswith('describe ') %}"
|
||||
"describe<|md_reserved_1|>{{ content['text'][9:] }}<|md_reserved_2|>"
|
||||
"{% else %}"
|
||||
"query<|md_reserved_1|>{{ content['text'] }}<|md_reserved_2|>"
|
||||
"{% endif %}"
|
||||
"{% endif %}"
|
||||
"{% endfor %}"
|
||||
"{% endif %}"
|
||||
"{% elif message['role'] == 'assistant' %}"
|
||||
"{{ message['content'] }}"
|
||||
"{% endif %}"
|
||||
"{% endfor %}"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: PreTrainedTokenizerBase | None = None,
|
||||
chat_template: str | None = None,
|
||||
crop_size: int = 378,
|
||||
max_crops: int = 12,
|
||||
overlap_margin: int = 4,
|
||||
patch_size: int = 14,
|
||||
**kwargs,
|
||||
):
|
||||
self.image_token = "<image>"
|
||||
self.crop_size = crop_size
|
||||
self.max_crops = max_crops
|
||||
self.overlap_margin = overlap_margin
|
||||
self.patch_size = patch_size
|
||||
|
||||
# Number of patches per crop (27x27 = 729 for 378/14)
|
||||
self.patches_per_crop = (crop_size // patch_size) ** 2
|
||||
|
||||
# Use default chat template if none provided
|
||||
if chat_template is None:
|
||||
chat_template = self._default_chat_template
|
||||
|
||||
super().__init__(tokenizer, chat_template=chat_template)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(
|
||||
cls,
|
||||
pretrained_model_name_or_path,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Load the processor, using a separate tokenizer repo.
|
||||
|
||||
The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1'
|
||||
instead of having tokenizer files in the model repo.
|
||||
"""
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizerFast
|
||||
from transformers.utils import cached_file
|
||||
|
||||
tokenizer = kwargs.pop("tokenizer", None)
|
||||
|
||||
tokenizer_kwargs = {
|
||||
"trust_remote_code": kwargs.get("trust_remote_code", False),
|
||||
}
|
||||
for key in (
|
||||
"cache_dir",
|
||||
"force_download",
|
||||
"local_files_only",
|
||||
"revision",
|
||||
"subfolder",
|
||||
"token",
|
||||
"use_fast",
|
||||
):
|
||||
if key in kwargs:
|
||||
tokenizer_kwargs[key] = kwargs[key]
|
||||
|
||||
cached_file_kwargs = {
|
||||
key: tokenizer_kwargs[key]
|
||||
for key in (
|
||||
"cache_dir",
|
||||
"force_download",
|
||||
"local_files_only",
|
||||
"revision",
|
||||
"subfolder",
|
||||
"token",
|
||||
)
|
||||
if key in tokenizer_kwargs
|
||||
}
|
||||
|
||||
def load_tokenizer(repo_or_path):
|
||||
try:
|
||||
return AutoTokenizer.from_pretrained(repo_or_path, **tokenizer_kwargs)
|
||||
except Exception:
|
||||
tokenizer_file = cached_file(
|
||||
repo_or_path,
|
||||
"tokenizer.json",
|
||||
**cached_file_kwargs,
|
||||
)
|
||||
return PreTrainedTokenizerFast(
|
||||
tokenizer_file=tokenizer_file,
|
||||
clean_up_tokenization_spaces=False,
|
||||
)
|
||||
|
||||
if isinstance(tokenizer, str):
|
||||
tokenizer = load_tokenizer(tokenizer)
|
||||
|
||||
if tokenizer is None:
|
||||
# Prefer model-local tokenizer files first. If unavailable, fall
|
||||
# back to moondream's dedicated tokenizer repository.
|
||||
try:
|
||||
tokenizer = load_tokenizer(pretrained_model_name_or_path)
|
||||
except Exception:
|
||||
tokenizer = load_tokenizer(cls._tokenizer_repo)
|
||||
|
||||
# Configure special tokens for Moondream3
|
||||
# BOS and EOS are both token 0 (<|endoftext|>), matching the native
|
||||
# config (TokenizerConfig.bos_id=0, eos_id=0). This is standard for
|
||||
# GPT-2 style models where <|endoftext|> signals both start and end.
|
||||
# Token 1 (<|md_reserved_0|>) is a template delimiter, NOT the EOS.
|
||||
tokenizer.bos_token = "<|endoftext|>"
|
||||
tokenizer.bos_token_id = 0
|
||||
tokenizer.eos_token = "<|endoftext|>"
|
||||
tokenizer.eos_token_id = 0
|
||||
|
||||
# Extract processor-specific kwargs
|
||||
crop_size = kwargs.pop("crop_size", 378)
|
||||
max_crops = kwargs.pop("max_crops", 12)
|
||||
overlap_margin = kwargs.pop("overlap_margin", 4)
|
||||
patch_size = kwargs.pop("patch_size", 14)
|
||||
chat_template = kwargs.pop("chat_template", None)
|
||||
|
||||
# Set default chat template on tokenizer if not already set
|
||||
if chat_template is None:
|
||||
chat_template = cls._default_chat_template
|
||||
if tokenizer.chat_template is None:
|
||||
tokenizer.chat_template = chat_template
|
||||
|
||||
return cls(
|
||||
tokenizer=tokenizer,
|
||||
chat_template=chat_template,
|
||||
crop_size=crop_size,
|
||||
max_crops=max_crops,
|
||||
overlap_margin=overlap_margin,
|
||||
patch_size=patch_size,
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: ImageInput = None,
|
||||
text: TextInput
|
||||
| PreTokenizedInput
|
||||
| list[TextInput]
|
||||
| list[PreTokenizedInput] = None,
|
||||
**kwargs: Unpack[Moondream3ProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Process images and text for Moondream3 model.
|
||||
|
||||
Args:
|
||||
images: Input images (PIL Image, numpy array, or list thereof).
|
||||
text: Input text or list of texts.
|
||||
**kwargs: Additional processing arguments.
|
||||
|
||||
Returns:
|
||||
BatchFeature with processed inputs.
|
||||
"""
|
||||
output_kwargs = self._merge_kwargs(
|
||||
Moondream3ProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Process images
|
||||
image_features = {}
|
||||
if images is not None:
|
||||
processed_images = []
|
||||
tilings = []
|
||||
|
||||
images_list = images if isinstance(images, list) else [images]
|
||||
for image in images_list:
|
||||
pixel_values, tiling = self.preprocess_image(
|
||||
image, **output_kwargs["images_kwargs"]
|
||||
)
|
||||
processed_images.append(pixel_values)
|
||||
tilings.append(tiling)
|
||||
|
||||
if processed_images:
|
||||
image_features["pixel_values"] = processed_images
|
||||
image_features["tilings"] = tilings
|
||||
|
||||
# Process text
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
# Get text kwargs, remove keys we set ourselves
|
||||
text_kwargs = output_kwargs.get("text_kwargs", {}).copy()
|
||||
text_kwargs.pop("return_tensors", None)
|
||||
text_kwargs.pop("add_special_tokens", None)
|
||||
|
||||
# Tokenize text
|
||||
tokenized = self.tokenizer(
|
||||
text,
|
||||
add_special_tokens=True,
|
||||
return_tensors="pt",
|
||||
**text_kwargs,
|
||||
)
|
||||
|
||||
output = BatchFeature(data=dict(tokenized))
|
||||
|
||||
# Add image features
|
||||
if image_features:
|
||||
output["pixel_values"] = image_features["pixel_values"]
|
||||
output["tilings"] = image_features["tilings"]
|
||||
|
||||
return output
|
||||
|
||||
# If only images were provided
|
||||
return BatchFeature(data=image_features)
|
||||
|
||||
@staticmethod
|
||||
def _image_array_to_uint8(array: np.ndarray) -> np.ndarray:
|
||||
if array.dtype == np.uint8:
|
||||
return np.ascontiguousarray(array)
|
||||
|
||||
if array.dtype == np.bool_:
|
||||
return np.ascontiguousarray(array.astype(np.uint8) * 255)
|
||||
|
||||
if np.issubdtype(array.dtype, np.floating):
|
||||
array = np.nan_to_num(array, nan=0.0, posinf=255.0, neginf=0.0)
|
||||
if array.size > 0 and array.max() <= 1.0:
|
||||
array = array * 255.0
|
||||
array = np.rint(array)
|
||||
|
||||
return np.ascontiguousarray(np.clip(array, 0, 255).astype(np.uint8))
|
||||
|
||||
@staticmethod
|
||||
def _to_pil_image(image: ImageInput) -> Image.Image:
|
||||
if isinstance(image, Image.Image):
|
||||
return image
|
||||
|
||||
if isinstance(image, torch.Tensor):
|
||||
tensor = image.detach().cpu()
|
||||
if tensor.dtype == torch.bfloat16:
|
||||
tensor = tensor.to(torch.float32)
|
||||
image_array = tensor.numpy()
|
||||
elif isinstance(image, np.ndarray):
|
||||
image_array = image
|
||||
else:
|
||||
raise TypeError(
|
||||
"Moondream3 images must be PIL images, numpy arrays, "
|
||||
f"or torch tensors, got {type(image)!r}."
|
||||
)
|
||||
|
||||
if image_array.ndim == 2:
|
||||
image_array = Moondream3Processor._image_array_to_uint8(image_array)
|
||||
return Image.fromarray(image_array)
|
||||
|
||||
if image_array.ndim != 3:
|
||||
raise ValueError(
|
||||
"Moondream3 image arrays must have 2 or 3 dimensions, "
|
||||
f"got shape {image_array.shape}."
|
||||
)
|
||||
|
||||
channel_dims = (1, 3, 4)
|
||||
if image_array.shape[-1] not in channel_dims:
|
||||
if image_array.shape[0] not in channel_dims:
|
||||
raise ValueError(
|
||||
"Moondream3 image arrays must be HWC or CHW with 1, 3, "
|
||||
f"or 4 channels, got shape {image_array.shape}."
|
||||
)
|
||||
image_array = np.transpose(image_array, (1, 2, 0))
|
||||
|
||||
image_array = Moondream3Processor._image_array_to_uint8(image_array)
|
||||
if image_array.shape[-1] == 1:
|
||||
image_array = image_array[..., 0]
|
||||
|
||||
return Image.fromarray(image_array)
|
||||
|
||||
def preprocess_image(
|
||||
self,
|
||||
image: ImageInput,
|
||||
max_crops: int = 12,
|
||||
overlap_margin: int = 4,
|
||||
crop_size: int = 378,
|
||||
patch_size: int = 14,
|
||||
convert_to_rgb: bool = True,
|
||||
return_tensors: str = "pt",
|
||||
) -> tuple[torch.Tensor, tuple[int, int]]:
|
||||
"""
|
||||
Preprocess an image using overlap-and-resize cropping strategy.
|
||||
|
||||
Args:
|
||||
image: Input PIL image, numpy array, or torch tensor.
|
||||
max_crops: Maximum number of crops.
|
||||
overlap_margin: Margin for overlapping in patches.
|
||||
crop_size: Size of each crop.
|
||||
patch_size: Size of each patch.
|
||||
convert_to_rgb: Whether to convert to RGB.
|
||||
return_tensors: Return type ("pt" for PyTorch).
|
||||
|
||||
Returns:
|
||||
Tuple of (pixel_values tensor, tiling tuple).
|
||||
"""
|
||||
image = self._to_pil_image(image)
|
||||
if convert_to_rgb:
|
||||
image = convert_image_mode(image, "RGB")
|
||||
|
||||
# Convert to numpy array
|
||||
image_array = np.array(image)
|
||||
original_h, original_w = image_array.shape[:2]
|
||||
|
||||
margin_pixels = patch_size * overlap_margin
|
||||
total_margin_pixels = margin_pixels * 2
|
||||
|
||||
crop_patches = crop_size // patch_size
|
||||
crop_window_patches = crop_patches - (2 * overlap_margin)
|
||||
crop_window_size = crop_window_patches * patch_size
|
||||
|
||||
tiling = select_tiling(
|
||||
original_h - total_margin_pixels,
|
||||
original_w - total_margin_pixels,
|
||||
crop_window_size,
|
||||
max_crops,
|
||||
)
|
||||
|
||||
n_crops = tiling[0] * tiling[1] + 1
|
||||
crops = np.zeros((n_crops, crop_size, crop_size, 3), dtype=np.uint8)
|
||||
|
||||
target_size = (
|
||||
tiling[0] * crop_window_size + total_margin_pixels,
|
||||
tiling[1] * crop_window_size + total_margin_pixels,
|
||||
)
|
||||
|
||||
# Resize image
|
||||
pil_img = Image.fromarray(image_array)
|
||||
resized = pil_img.resize(
|
||||
(int(target_size[1]), int(target_size[0])),
|
||||
resample=Image.Resampling.LANCZOS,
|
||||
)
|
||||
resized_array = np.asarray(resized)
|
||||
|
||||
# Create global crop
|
||||
global_pil = pil_img.resize(
|
||||
(crop_size, crop_size), resample=Image.Resampling.LANCZOS
|
||||
)
|
||||
crops[0] = np.asarray(global_pil)
|
||||
|
||||
# Create local crops
|
||||
for i in range(tiling[0]):
|
||||
for j in range(tiling[1]):
|
||||
y0 = i * crop_window_size
|
||||
x0 = j * crop_window_size
|
||||
y_end = min(y0 + crop_size, resized_array.shape[0])
|
||||
x_end = min(x0 + crop_size, resized_array.shape[1])
|
||||
|
||||
crop_region = resized_array[y0:y_end, x0:x_end]
|
||||
crop_idx = 1 + i * tiling[1] + j
|
||||
h_slice = slice(None, crop_region.shape[0])
|
||||
w_slice = slice(None, crop_region.shape[1])
|
||||
crops[crop_idx, h_slice, w_slice] = crop_region
|
||||
|
||||
# Convert to tensor: (n_crops, H, W, C) -> (n_crops, C, H, W)
|
||||
pixel_values = np.transpose(crops, (0, 3, 1, 2))
|
||||
|
||||
if return_tensors == "pt":
|
||||
# Match HF reference preprocessing exactly: convert uint8 crops to
|
||||
# bfloat16 before in-place normalization.
|
||||
pixel_values = (
|
||||
torch.from_numpy(pixel_values)
|
||||
.to(dtype=torch.bfloat16)
|
||||
.div_(255.0)
|
||||
.sub_(0.5)
|
||||
.div_(0.5)
|
||||
)
|
||||
else:
|
||||
pixel_values = pixel_values.astype(np.float32) / 255.0
|
||||
pixel_values = (pixel_values - 0.5) / 0.5
|
||||
|
||||
return pixel_values, tiling
|
||||
|
||||
def get_num_image_tokens(self) -> int:
|
||||
"""Return the number of image tokens (729 = 27x27 patches)."""
|
||||
return self.patches_per_crop
|
||||
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""Forward to tokenizer's batch_decode."""
|
||||
return self.tokenizer.batch_decode(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
"""Forward to tokenizer's decode."""
|
||||
return self.tokenizer.decode(*args, **kwargs)
|
||||
|
||||
@property
|
||||
def model_input_names(self):
|
||||
tokenizer_input_names = self.tokenizer.model_input_names
|
||||
return tokenizer_input_names + ["pixel_values", "tilings"]
|
||||
|
||||
|
||||
AutoProcessor.register("Moondream3Processor", Moondream3Processor)
|
||||
Reference in New Issue
Block a user