Add Cosmos3 Edge Reasoner model (#48291)

Signed-off-by: Bartosz Stefaniak <bstefaniak@nvidia.com>
Co-authored-by: Bartosz Stefaniak <bstefaniak@nvidia.com>
This commit is contained in:
adsridhar
2026-07-14 08:14:50 +00:00
committed by GitHub
co-authored by Bartosz Stefaniak
parent c9a788eedc
commit 894ebb27f5
13 changed files with 1474 additions and 27 deletions
+1
View File
@@ -514,6 +514,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `CheersForConditionalGeneration` | Cheers | T + I | `ai9stars/Cheers` | | ✅︎ |
| `Cohere2VisionForConditionalGeneration` | Command A Vision, Command-A+ | T + I<sup>+</sup> | `CohereLabs/command-a-vision-07-2025`, `CohereLabs/command-a-plus-05-2026`, etc. | | ✅︎ |
| `Cosmos3ForConditionalGeneration` | Cosmos3 (understanding tower) | T + I<sup>E+</sup> + V<sup>E+</sup> | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | | ✅︎ |
| `Cosmos3EdgeForConditionalGeneration` | Cosmos3-Edge (understanding tower) | T + I<sup>E+</sup> + V<sup>E+</sup> | `nvidia/Cosmos3-Edge` | | ✅︎ |
| `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
@@ -0,0 +1,101 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import pytest
from vllm.assets.video import VideoAsset
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
MODEL_ID = "nvidia/Cosmos3-Edge"
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
LOCAL_MODEL_PATH = os.getenv("COSMOS3_EDGE_MODEL_PATH")
@pytest.fixture(scope="module")
def processor():
if LOCAL_MODEL_PATH:
model_config = ModelConfig(
LOCAL_MODEL_PATH,
tokenizer=LOCAL_MODEL_PATH,
max_model_len=4096,
limit_mm_per_prompt={"image": 2, "video": 1},
)
else:
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 2, "video": 1},
)
model_config = ctx.model_config
return MULTIMODAL_REGISTRY.create_processor(model_config)
def _assert_image_outputs(processor, processed, num_images: int) -> None:
mm_data = processed["mm_kwargs"].get_data()
grid_thw = mm_data["image_grid_thw"]
pixel_values = mm_data["pixel_values"]
assert grid_thw.shape == (num_images, 3)
assert pixel_values.shape[0] == int(grid_thw.prod(dim=-1).sum())
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
expected_tokens = (grid_thw.prod(dim=-1) // merge_size**2).tolist()
image_placeholders = processed["mm_placeholders"]["image"]
assert len(image_placeholders) == num_images
assert [placeholder.length for placeholder in image_placeholders] == (
expected_tokens
)
image_token_id = processor.info.get_hf_processor().image_token_id
assert processed["prompt_token_ids"].count(image_token_id) == sum(expected_tokens)
def _assert_video_outputs(processor, processed) -> None:
mm_data = processed["mm_kwargs"].get_data()
grid_thw = mm_data["video_grid_thw"]
pixel_values = mm_data["pixel_values_videos"]
assert grid_thw.shape == (1, 3)
assert pixel_values.shape[0] == int(grid_thw.prod())
assert len(processed["mm_placeholders"]["video"]) == 1
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
expected_tokens = int(grid_thw.prod()) // merge_size**2
video_token_id = processor.info.get_hf_config().video_token_id
assert processed["prompt_token_ids"].count(video_token_id) == expected_tokens
@pytest.mark.parametrize("num_images", [1, 2])
def test_process_images(
processor,
image_assets: ImageTestAssets,
num_images: int,
) -> None:
images = [asset.pil_image for asset in image_assets[:num_images]]
processed = processor(
IMAGE_PLACEHOLDER * num_images,
mm_items=processor.info.parse_mm_data({"image": images}),
hf_processor_mm_kwargs={},
)
_assert_image_outputs(processor, processed, num_images)
def test_process_video(processor) -> None:
video_asset = VideoAsset(name="baby_reading", num_frames=8)
video = (video_asset.np_ndarrays, video_asset.metadata)
processed = processor(
VIDEO_PLACEHOLDER,
mm_items=processor.info.parse_mm_data({"video": [video]}),
hf_processor_mm_kwargs={},
)
_assert_video_outputs(processor, processed)
+71
View File
@@ -74,6 +74,77 @@ def test_cosmos3_new_checkpoint_weights_mapper():
)
def test_cosmos3_edge_checkpoint_weights_mapper():
from vllm.model_executor.models.cosmos3_edge import (
Cosmos3EdgeForConditionalGeneration,
)
mapper = Cosmos3EdgeForConditionalGeneration.hf_to_vllm_mapper
assert mapper.apply_list(
[
"embed_tokens.weight",
"norm.weight",
"layers.0.input_layernorm.weight",
"layers.0.self_attn.to_q.weight",
"layers.0.self_attn.to_k.weight",
"layers.0.self_attn.to_v.weight",
"layers.0.self_attn.to_out.weight",
"layers.0.post_attention_layernorm.weight",
"layers.0.mlp.up_proj.weight",
"layers.0.mlp.down_proj.weight",
"layers.27.input_layernorm.weight",
"layers.27.self_attn.to_q.weight",
"layers.27.post_attention_layernorm.weight",
"layers.27.mlp.down_proj.weight",
"model.visual.embeddings.patch_embedding.weight",
"model.visual.encoder.layers.0.self_attn.q_proj.weight",
"model.projector.linear_fc1.weight",
"lm_head.weight",
]
) == [
"language_model.model.embed_tokens.weight",
"language_model.model.norm_f.weight",
"language_model.model.layers.0.norm.weight",
"language_model.model.layers.0.mixer.qkv_proj.weight",
"language_model.model.layers.0.mixer.qkv_proj.weight",
"language_model.model.layers.0.mixer.qkv_proj.weight",
"language_model.model.layers.0.mixer.o_proj.weight",
"language_model.model.layers.1.norm.weight",
"language_model.model.layers.1.mixer.up_proj.weight",
"language_model.model.layers.1.mixer.down_proj.weight",
"language_model.model.layers.54.norm.weight",
"language_model.model.layers.54.mixer.qkv_proj.weight",
"language_model.model.layers.55.norm.weight",
"language_model.model.layers.55.mixer.down_proj.weight",
"visual.encoder.embeddings.patch_embedding.weight",
"visual.encoder.encoder.layers.0.self_attn.qkv_proj.weight",
"visual.projector.linear_fc1.weight",
"language_model.lm_head.weight",
]
assert (
mapper.apply_list(
[
"layers.0.self_attn.add_q_proj.weight",
"layers.0.self_attn.add_k_proj.weight",
"layers.0.self_attn.add_v_proj.weight",
"layers.0.self_attn.to_add_out.weight",
"layers.0.self_attn.norm_added_q.weight",
"layers.0.self_attn.norm_added_k.weight",
"layers.0.self_attn.q_proj_moe_gen.weight",
"layers.0.mlp_moe_gen.up_proj.weight",
"norm_moe_gen.weight",
"proj_in.weight",
"proj_out.weight",
"audio_modality_embed",
"action_modality_embed",
]
)
== []
)
def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
"""Create weights from safetensors checkpoint metadata"""
metadata = try_get_safetensors_metadata(repo)
+5
View File
@@ -811,6 +811,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
max_model_len=4096,
min_transformers_version="4.57",
),
"Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo(
"nvidia/Cosmos3-Edge",
max_model_len=4096,
is_available_online=False,
),
"DeepseekVLV2ForCausalLM": _HfExamplesInfo(
"deepseek-ai/deepseek-vl2-tiny",
extras={"fork": "Isotr0py/deepseek-vl2-tiny"},
+4 -21
View File
@@ -3,8 +3,6 @@
import regex
from vllm.config import VllmConfig
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
from vllm.model_executor.models.utils import WeightsMapper
@@ -52,22 +50,7 @@ class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration):
},
)
allow_patterns_overrides = ["transformer/*.safetensors"]
"""
Cosmos3 checkpoint separates transformer weights and vision_encoder weights
into separate directories, as it's in diffusers checkpoint format.
Using secondary_weights here to load all necessary weights for
the Reasoner-only part.
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__(vllm_config=vllm_config, prefix=prefix)
self.secondary_weights = [
DefaultModelLoader.Source(
model_or_path=vllm_config.model_config.model,
revision=vllm_config.model_config.revision,
prefix="",
allow_patterns_overrides=["vision_encoder/*.safetensors"],
),
]
# Cosmos3 unified Diffusers checkpoints store reasoner weights across
# transformer/ and vision_encoder/. Match both while excluding VAE and
# sound_tokenizer weights; the mapper drops generation-only tensors.
allow_patterns_overrides = ["[tv]*er/*.safetensors"]
+751
View File
@@ -0,0 +1,751 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
import torch
import torch.nn as nn
from transformers import ProcessorMixin
from vllm.config import VllmConfig
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
from .interfaces import (
MultiModalEmbeddings,
SupportsMRoPE,
SupportsMultiModal,
SupportsPP,
)
from .lfm2_siglip2 import Siglip2VisionTransformer
from .nemotron_h import (
NemotronHAttention,
NemotronHMLPDecoderLayer,
NemotronHModel,
)
from .qwen2_5_vl import (
Qwen2_5_VLImageEmbeddingInputs,
Qwen2_5_VLImageInputs,
Qwen2_5_VLImagePixelInputs,
Qwen2_5_VLVideoEmbeddingInputs,
Qwen2_5_VLVideoInputs,
Qwen2_5_VLVideoPixelInputs,
)
from .qwen3_vl import (
Qwen3VLDummyInputsBuilder,
Qwen3VLForConditionalGeneration,
Qwen3VLMultiModalProcessor,
Qwen3VLProcessingInfo,
)
from .utils import (
AutoWeightsLoader,
WeightsMapper,
maybe_prefix,
)
from .vision import (
is_vit_use_data_parallel,
run_dp_sharded_mrope_vision_model,
)
class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer):
"""Adapts Cosmos (T, H, W) metadata to vLLM packed SigLIP2."""
@property
def dtype(self) -> torch.dtype:
return self.embeddings.patch_embedding.weight.dtype
def forward(
self,
pixel_values: torch.Tensor,
grid_thw: torch.Tensor,
) -> torch.Tensor:
# SigLIP2 attends to each frame independently, so expand every THW
# entry into T separate HW attention sequences.
grid_thw_cpu = grid_thw.to(device="cpu")
spatial_shapes = torch.repeat_interleave(
grid_thw_cpu[:, 1:],
grid_thw_cpu[:, 0],
dim=0,
)
lengths_cpu = spatial_shapes.prod(dim=-1).to(torch.int32)
lengths = lengths_cpu.to(
device=pixel_values.device,
non_blocking=True,
)
cu_seqlens = torch.zeros(
lengths.numel() + 1,
dtype=torch.int32,
device=pixel_values.device,
)
cu_seqlens[1:] = lengths.cumsum(dim=0)
max_seqlen = lengths_cpu.max().reshape(1)
return super().forward(
pixel_values_packed=pixel_values,
spatial_shapes=spatial_shapes,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
)[0]
def patch_merging_by_param(
image_embeds: torch.Tensor,
grid_thw: torch.Tensor,
merge_size: int = 2,
) -> torch.Tensor:
"""Merge each spatial ``merge_size`` block into the channel dimension."""
merged_embeds: list[torch.Tensor] = []
current_idx = 0
hidden_size = image_embeds.shape[-1]
for grid in grid_thw:
t, h, w = grid.tolist()
if h % merge_size != 0 or w % merge_size != 0:
raise ValueError(
"Grid height and width must be divisible by merge_size: "
f"got grid={(t, h, w)} and merge_size={merge_size}."
)
num_patches = t * h * w
media_embeds = image_embeds[current_idx : current_idx + num_patches]
if media_embeds.shape[0] != num_patches:
raise ValueError(
"image_embeds contains fewer patches than grid_thw describes."
)
current_idx += num_patches
media_embeds = media_embeds.view(t, h, w, hidden_size)
media_embeds = media_embeds.view(
t,
h // merge_size,
merge_size,
w // merge_size,
merge_size,
hidden_size,
)
media_embeds = media_embeds.permute(0, 1, 3, 2, 4, 5).contiguous()
merged_embeds.append(
media_embeds.view(-1, merge_size * merge_size * hidden_size)
)
if current_idx != image_embeds.shape[0]:
raise ValueError(
"image_embeds contains more patches than grid_thw describes: "
f"got {image_embeds.shape[0]}, expected {current_idx}."
)
if not merged_embeds:
return image_embeds.new_empty((0, merge_size * merge_size * hidden_size))
return torch.cat(merged_embeds, dim=0)
class Cosmos3EdgePatchMerger(nn.Module):
"""
Projector: LayerNorm -> Linear -> GELU -> Linear
Reads config from projector_config (not vision_config).
input_hidden_size * spatial_merge_size² -> merger_intermediate_size
-> out_hidden_size
"""
def __init__(
self,
input_hidden_size: int,
out_hidden_size: int,
merger_intermediate_size: int,
spatial_merge_size: int = 2,
use_postshuffle_norm: bool = False,
quant_config=None,
prefix: str = "",
) -> None:
super().__init__()
use_data_parallel = is_vit_use_data_parallel()
self.spatial_merge_size = spatial_merge_size
self.hidden_size = input_hidden_size * (spatial_merge_size**2)
self.input_hidden_size = input_hidden_size
self.out_hidden_size = out_hidden_size
self.use_postshuffle_norm = use_postshuffle_norm
norm_dim = self.hidden_size if use_postshuffle_norm else input_hidden_size
self.norm = nn.LayerNorm(norm_dim, eps=1e-6)
self.linear_fc1 = ColumnParallelLinear(
self.hidden_size,
merger_intermediate_size,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.linear_fc1",
disable_tp=use_data_parallel,
)
self.act_fn = nn.GELU()
self.linear_fc2 = RowParallelLinear(
merger_intermediate_size,
out_hidden_size,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.linear_fc2",
disable_tp=use_data_parallel,
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.use_postshuffle_norm:
x = self.norm(x.view(-1, self.hidden_size))
else:
x = self.norm(x).view(-1, self.hidden_size)
x, _ = self.linear_fc1(x)
x = self.act_fn(x)
x, _ = self.linear_fc2(x)
return x
class Cosmos3EdgeVisionModel(nn.Module):
"""Complete Cosmos vision tower returning language-model embeddings."""
def __init__(
self,
vision_config,
projector_config,
quant_config=None,
prefix: str = "",
) -> None:
super().__init__()
self.spatial_merge_size = projector_config.spatial_merge_size
self.out_hidden_size = projector_config.out_hidden_size
self.encoder = Cosmos3EdgeVisionEncoder(
vision_config,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "encoder"),
)
self.projector = Cosmos3EdgePatchMerger(
input_hidden_size=projector_config.input_hidden_size,
out_hidden_size=projector_config.out_hidden_size,
merger_intermediate_size=projector_config.merger_intermediate_size,
spatial_merge_size=self.spatial_merge_size,
use_postshuffle_norm=getattr(
projector_config, "use_postshuffle_norm", False
),
quant_config=quant_config,
prefix=maybe_prefix(prefix, "projector"),
)
@property
def dtype(self) -> torch.dtype:
return self.encoder.dtype
def forward(
self,
pixel_values: torch.Tensor,
grid_thw: torch.Tensor | list[list[int]],
) -> torch.Tensor:
grid_thw = torch.as_tensor(grid_thw, dtype=torch.int64, device="cpu")
image_embeds = self.encoder(pixel_values.type(self.dtype), grid_thw=grid_thw)
image_embeds = patch_merging_by_param(
image_embeds,
grid_thw,
merge_size=self.spatial_merge_size,
)
image_embeds = image_embeds.view(
-1,
self.spatial_merge_size**2,
self.projector.input_hidden_size,
)
return self.projector(image_embeds)
class Cosmos3EdgeProcessingInfo(Qwen3VLProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config()
def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
return self.ctx.get_hf_processor(**kwargs)
class Cosmos3EdgeMultiModalProcessor(Qwen3VLMultiModalProcessor):
pass
class Cosmos3EdgeDummyInputsBuilder(Qwen3VLDummyInputsBuilder):
pass
class Cosmos3EdgeAttention(NemotronHAttention):
"""Nemotron-H attention with interleaved multimodal RoPE."""
def __init__(
self,
config: NemotronHConfig,
layer_idx: int,
model_config=None,
cache_config=None,
quant_config=None,
prefix: str = "",
) -> None:
super().__init__(
config,
layer_idx,
model_config,
cache_config,
quant_config,
prefix,
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=config.max_position_embeddings,
rope_parameters=config.rope_parameters,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
**kwargs,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.o_proj(attn_output)
return output
class Cosmos3EdgeAttentionDecoderLayer(nn.Module):
"""Pre-norm attention layer for the Cosmos3 Edge dense text model."""
def __init__(
self,
config: NemotronHConfig,
layer_idx: int,
model_config=None,
cache_config=None,
quant_config=None,
parallel_config=None,
prefix: str = "",
) -> None:
super().__init__()
get_layer_config = getattr(config, "get_nemotron_h_config_for_layer", None)
layer_config = get_layer_config(layer_idx) if get_layer_config else config
self.mixer = Cosmos3EdgeAttention(
layer_config,
layer_idx,
model_config,
cache_config,
quant_config,
prefix=f"{prefix}.mixer",
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is None:
residual = hidden_states
hidden_states = self.norm(hidden_states)
else:
hidden_states, residual = self.norm(hidden_states, residual)
hidden_states = self.mixer(positions=positions, hidden_states=hidden_states)
return hidden_states, residual
class Cosmos3EdgeTextModel(NemotronHModel):
"""Nemotron-H backbone with Cosmos-owned checkpoint name mapping."""
# The enclosing Cosmos model maps and packs checkpoint weights. Disabling
# Nemotron-H's checkpoint-specific hook prevents those names from being
# remapped a second time when AutoWeightsLoader descends into this module.
load_weights = None # type: ignore[assignment]
class Cosmos3EdgeForCausalLM(nn.Module):
"""Minimal CausalLM wrapper for the Cosmos3 Edge dense text model."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
hf_config = vllm_config.model_config.hf_config
text_config = getattr(hf_config, "text_config", hf_config)
vllm_config = vllm_config.with_hf_config(text_config)
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
self.quant_config = vllm_config.quant_config
self.config = text_config
pattern = text_config.hybrid_override_pattern
unsupported = set(pattern) - {"*", "-"}
if unsupported:
raise ValueError(
"Cosmos3 Edge only supports attention (`*`) and MLP (`-`) "
f"layers, but found: {sorted(unsupported)}"
)
if len(pattern) != text_config.num_hidden_layers:
raise ValueError(
"hybrid_override_pattern must contain one entry per layer: "
f"expected {text_config.num_hidden_layers}, "
f"got {len(pattern)}"
)
self.model = Cosmos3EdgeTextModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
decoder_layer_types={
"*": Cosmos3EdgeAttentionDecoderLayer,
"-": NemotronHMLPDecoderLayer,
},
)
self.lm_head = ParallelLMHead(
text_config.vocab_size,
text_config.hidden_size,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(text_config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
):
return self.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def _cosmos3_edge_diffusers_prefix_map() -> dict[str, str]:
"""Map Diffusers blocks to interleaved Nemotron-H layers.
The checkpoint stores attention and MLP in one ``layers.N`` block, while
the Nemotron-H model represents them as separate decoder layers. Therefore,
checkpoint block ``N`` maps its attention to layer ``2N`` and its MLP to
layer ``2N + 1``.
"""
mappings = {
"embed_tokens.": "language_model.model.embed_tokens.",
"norm.": "language_model.model.norm_f.",
}
for physical_idx in range(28):
attention_idx = 2 * physical_idx
mlp_idx = attention_idx + 1
source_prefix = f"layers.{physical_idx}"
attention_prefix = f"language_model.model.layers.{attention_idx}"
mlp_prefix = f"language_model.model.layers.{mlp_idx}"
mappings.update(
{
f"{source_prefix}.input_layernorm.": f"{attention_prefix}.norm.",
f"{source_prefix}.self_attn.qkv_proj.": (
f"{attention_prefix}.mixer.qkv_proj."
),
f"{source_prefix}.self_attn.o_proj.": (
f"{attention_prefix}.mixer.o_proj."
),
f"{source_prefix}.post_attention_layernorm.": (f"{mlp_prefix}.norm."),
f"{source_prefix}.mlp.": f"{mlp_prefix}.mixer.",
}
)
return mappings
@MULTIMODAL_REGISTRY.register_processor(
Cosmos3EdgeMultiModalProcessor,
info=Cosmos3EdgeProcessingInfo,
dummy_inputs=Cosmos3EdgeDummyInputsBuilder,
)
class Cosmos3EdgeForConditionalGeneration(
nn.Module,
SupportsMultiModal,
SupportsPP,
SupportsMRoPE,
):
"""
Cosmos3 Edge model with a SigLIP2 vision encoder.
Architecture:
- self.visual: SigLIP2 encoder + patch merger + projector
- self.language_model: Cosmos3EdgeForCausalLM (pure attention + RoPE)
"""
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
},
orig_to_new_prefix={
**_cosmos3_edge_diffusers_prefix_map(),
"proj_in.": None,
"proj_out.": None,
"time_embedder.": None,
"action_proj_in.": None,
"action_proj_out.": None,
"audio_modality_embed": None,
"action_modality_embed": None,
"model.visual.": "visual.encoder.",
"model.projector.": "visual.projector.",
"lm_head.": "language_model.lm_head.",
"model.language_model.": "language_model.model.",
},
orig_to_new_substr={
"_moe_gen": None,
".add_q_proj.": None,
".add_k_proj.": None,
".add_v_proj.": None,
".to_add_out.": None,
".norm_added_q.": None,
".norm_added_k.": None,
".self_attn.to_q.": ".self_attn.q_proj.",
".self_attn.to_k.": ".self_attn.k_proj.",
".self_attn.to_v.": ".self_attn.v_proj.",
".self_attn.to_out.": ".self_attn.o_proj.",
"language_model.embeddings": "language_model.embed_tokens",
},
)
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
}
# Cosmos3 Edge unified Diffusers checkpoints store reasoner weights across
# transformer/ and vision_encoder/. Match both while excluding VAE weights.
allow_patterns_overrides = ["[tv]*er/*.safetensors"]
supports_encoder_tp_data = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|vision_start|><|image_pad|><|vision_end|>"
if modality.startswith("video"):
return "<|vision_start|><|video_pad|><|vision_end|>"
raise ValueError(f"Unsupported modality: {modality}")
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
self.config = config
self.multimodal_config = multimodal_config
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
with self._mark_tower_model(vllm_config, {"image", "video"}):
self.visual = Cosmos3EdgeVisionModel(
vision_config=config.vision_config,
projector_config=config.projector_config,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "visual"),
)
with self._mark_language_model(vllm_config):
self.language_model = Cosmos3EdgeForCausalLM(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "language_model"),
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
def _get_image_features(
self,
pixel_values: torch.Tensor,
grid_thw: torch.Tensor,
) -> tuple[torch.Tensor, ...]:
"""Run the complete vision tower and split by media item."""
image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
sizes = (grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist()
return image_embeds.split(sizes)
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Qwen2_5_VLImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
image_grid_thw = kwargs.pop("image_grid_thw", None)
if pixel_values is None and image_embeds is None:
return None
if pixel_values is not None:
return Qwen2_5_VLImagePixelInputs(
type="pixel_values",
pixel_values=pixel_values,
image_grid_thw=image_grid_thw,
)
if image_embeds is not None:
return Qwen2_5_VLImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Qwen2_5_VLVideoInputs | None:
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
video_embeds = kwargs.pop("video_embeds", None)
video_grid_thw = kwargs.pop("video_grid_thw", None)
second_per_grid_ts = kwargs.pop("second_per_grid_ts", None)
if pixel_values_videos is None and video_embeds is None:
return None
if pixel_values_videos is not None:
return Qwen2_5_VLVideoPixelInputs(
type="pixel_values_videos",
pixel_values_videos=pixel_values_videos,
video_grid_thw=video_grid_thw,
second_per_grid_ts=second_per_grid_ts,
)
if video_embeds is not None:
return Qwen2_5_VLVideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
)
def _process_image_input(
self, image_input: Qwen2_5_VLImageInputs
) -> tuple[torch.Tensor, ...]:
grid_thw = image_input["image_grid_thw"]
assert grid_thw.ndim == 2
if image_input["type"] == "image_embeds":
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
merge_size = self.visual.spatial_merge_size
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
return image_embeds.split(sizes)
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
)
return self._get_image_features(pixel_values, grid_thw)
def _process_video_input(
self, video_input: Qwen2_5_VLVideoInputs
) -> tuple[torch.Tensor, ...]:
grid_thw = video_input["video_grid_thw"]
assert grid_thw.ndim == 2
if video_input["type"] == "video_embeds":
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
merge_size = self.visual.spatial_merge_size
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
return video_embeds.split(sizes)
pixel_values_videos = video_input["pixel_values_videos"].type(self.visual.dtype)
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual, pixel_values_videos, grid_thw.tolist(), rope_type="rope_3d"
)
return self._get_image_features(pixel_values_videos, grid_thw)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
modalities = {}
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is not None:
modalities["image"] = image_input
video_input = self._parse_and_validate_video_input(**kwargs)
if video_input is not None:
modalities["video"] = video_input
return modalities
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
if not mm_input_by_modality:
return None
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
for modality in mm_input_by_modality:
multimodal_input = mm_input_by_modality[modality]
if modality == "image":
image_embeddings = self._process_image_input(multimodal_input)
multimodal_embeddings += tuple(image_embeddings)
if modality == "video":
video_embeddings = self._process_video_input(multimodal_input)
multimodal_embeddings += tuple(video_embeddings)
return multimodal_embeddings
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
return self.language_model.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
return MultiModelKeys.from_string_field(
language_model="language_model",
connector="visual.projector",
tower_model="visual.",
)
def get_mrope_input_positions(
self,
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec],
) -> tuple[torch.Tensor, int]:
return Qwen3VLForConditionalGeneration._get_mrope_input_positions(
input_tokens=input_tokens,
mm_features=mm_features,
config=self.config,
)
+24 -6
View File
@@ -19,7 +19,7 @@
"""Inference-only NemotronH model."""
import typing
from collections.abc import Callable, Iterable
from collections.abc import Callable, Iterable, Mapping
from itertools import islice
import torch
@@ -540,9 +540,22 @@ ALL_DECODER_LAYER_TYPES = {
}
@support_torch_compile
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
}
)
class NemotronHModel(nn.Module, EagleModelMixin):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
def __init__(
self,
*,
vllm_config: VllmConfig,
prefix: str = "",
decoder_layer_types: Mapping[str, type[nn.Module]] | None = None,
):
super().__init__()
config: NemotronHConfig = vllm_config.model_config.hf_config
@@ -562,11 +575,16 @@ class NemotronHModel(nn.Module, EagleModelMixin):
self.has_moe = "E" in config.hybrid_override_pattern
layer_types = decoder_layer_types or ALL_DECODER_LAYER_TYPES
def get_layer(prefix: str):
layer_idx = int(prefix.rsplit(".", 1)[1])
layer_class = ALL_DECODER_LAYER_TYPES[
config.hybrid_override_pattern[layer_idx]
]
layer_type = config.hybrid_override_pattern[layer_idx]
if layer_type not in layer_types:
raise ValueError(
f"Unsupported layer type {layer_type!r} at layer {layer_idx}"
)
layer_class = layer_types[layer_type]
return layer_class(
config=config,
layer_idx=layer_idx,
+4
View File
@@ -349,6 +349,10 @@ _MULTIMODAL_MODELS = {
"Cohere2VisionForConditionalGeneration",
),
"Cosmos3ForConditionalGeneration": ("cosmos3", "Cosmos3ForConditionalGeneration"),
"Cosmos3EdgeForConditionalGeneration": (
"cosmos3_edge",
"Cosmos3EdgeForConditionalGeneration",
),
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
+1
View File
@@ -81,6 +81,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
ops_colqwen3="OpsColQwen3Config",
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
cosmos3_omni="Cosmos3Config",
cosmos3_edge="Cosmos3EdgeConfig",
diffusion_gemma="DiffusionGemmaConfig",
deepseek_vl_v2="DeepseekVLV2Config",
deepseek_v32="DeepseekV3Config",
@@ -27,6 +27,10 @@ _CLASS_TO_MODULE: dict[str, str] = {
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
"Cosmos3Config": "vllm.transformers_utils.configs.cosmos3",
"Cosmos3EdgeConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeProjectorConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"DiffusionGemmaConfig": "vllm.transformers_utils.configs.diffusion_gemma",
"DiffusionGemmaTextConfig": "vllm.transformers_utils.configs.diffusion_gemma",
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
@@ -107,6 +111,10 @@ __all__ = [
"OpsColQwen3Config",
"Qwen3VLNemotronEmbedConfig",
"Cosmos3Config",
"Cosmos3EdgeConfig",
"Cosmos3EdgeProjectorConfig",
"Cosmos3EdgeTextConfig",
"Cosmos3EdgeVisionConfig",
"DiffusionGemmaConfig",
"DiffusionGemmaTextConfig",
"DeepseekVLV2Config",
@@ -0,0 +1,146 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers.configuration_utils import PretrainedConfig
from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig
from .nemotron_h import NemotronHConfig
def _normalize_mrope_parameters(config: NemotronHConfig) -> None:
"""Normalize the checkpoint's mRoPE configuration for vLLM."""
rope_parameters = dict(getattr(config, "rope_parameters", None) or {})
mrope_section = rope_parameters.get("mrope_section")
if mrope_section is None:
return
rope_parameters["mrope_section"] = list(mrope_section)
rope_parameters.setdefault("mrope_interleaved", True)
config.rope_parameters = rope_parameters
class Cosmos3EdgeTextConfig(NemotronHConfig):
"""Dense Cosmos3 Edge text config backed by Nemotron-H layers."""
model_type = "cosmos3_edge_text"
has_no_defaults_at_init = True
ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"}
def __init__(
self,
num_hidden_layers: int = 28,
hidden_act: str = "relu2",
rms_norm_eps: float = 1e-5,
**kwargs,
) -> None:
super().__init__(
num_hidden_layers=2 * num_hidden_layers,
hybrid_override_pattern="*-" * num_hidden_layers,
mlp_hidden_act=hidden_act,
layer_norm_epsilon=rms_norm_eps,
**kwargs,
)
def to_dict(self) -> dict:
config_dict = super().to_dict()
config_dict["num_hidden_layers"] = self.num_hidden_layers // 2
config_dict["hidden_act"] = config_dict.pop("mlp_hidden_act")
config_dict["rms_norm_eps"] = config_dict.pop("layer_norm_epsilon")
config_dict.pop("hybrid_override_pattern", None)
return config_dict
class Cosmos3EdgeProjectorConfig(PretrainedConfig):
model_type = "cosmos3_edge_projector"
def __init__(
self,
input_hidden_size: int = 1024,
use_postshuffle_norm: bool = False,
spatial_merge_size: int = 2,
merger_intermediate_size: int = 11520,
out_hidden_size: int = 1024,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.input_hidden_size = input_hidden_size
self.use_postshuffle_norm = use_postshuffle_norm
self.spatial_merge_size = spatial_merge_size
self.merger_intermediate_size = merger_intermediate_size
self.out_hidden_size = out_hidden_size
class Cosmos3EdgeVisionConfig(Siglip2VisionConfig):
model_type = "cosmos3_edge_vision"
class Cosmos3EdgeConfig(PretrainedConfig):
model_type = "cosmos3_edge"
keys_to_ignore_at_inference = ["past_key_values"]
has_no_defaults_at_init = True
sub_configs = {
"vision_config": Cosmos3EdgeVisionConfig,
"projector_config": Cosmos3EdgeProjectorConfig,
"text_config": Cosmos3EdgeTextConfig,
}
def __init__(
self,
text_config: Cosmos3EdgeTextConfig | dict | None = None,
vision_config: Cosmos3EdgeVisionConfig | dict | None = None,
projector_config: Cosmos3EdgeProjectorConfig | dict | None = None,
image_token_id: int = 19,
video_token_id: int = 18,
vision_start_token_id: int = 20,
vision_end_token_id: int = 21,
tie_word_embeddings: bool = False,
**kwargs,
) -> None:
if text_config is None:
text_config = {}
if vision_config is None:
vision_config = {}
if projector_config is None:
projector_config = {}
self.text_config = (
Cosmos3EdgeTextConfig(**text_config)
if isinstance(text_config, dict)
else text_config
)
_normalize_mrope_parameters(self.text_config)
self.vision_config = (
Cosmos3EdgeVisionConfig(**vision_config)
if isinstance(vision_config, dict)
else vision_config
)
self.projector_config = (
Cosmos3EdgeProjectorConfig(**projector_config)
if isinstance(projector_config, dict)
else projector_config
)
# Qwen3-VL processing and M-RoPE read these attributes from the
# vision config, while this checkpoint stores them in projector config.
self.vision_config.spatial_merge_size = self.projector_config.spatial_merge_size
self.vision_config.temporal_patch_size = 1
self.vision_config.out_hidden_size = self.projector_config.out_hidden_size
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.vision_start_token_id = vision_start_token_id
self.vision_end_token_id = vision_end_token_id
super().__init__(
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
__all__ = [
"Cosmos3EdgeConfig",
"Cosmos3EdgeProjectorConfig",
"Cosmos3EdgeTextConfig",
"Cosmos3EdgeVisionConfig",
]
@@ -14,6 +14,7 @@ __all__ = [
"BagelProcessor",
"CheersProcessor",
"CohereASRProcessor",
"Cosmos3EdgeProcessor",
"DeepseekVLV2Processor",
"FireRedASR2Processor",
"FireRedLIDProcessor",
@@ -49,6 +50,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"BagelProcessor": "vllm.transformers_utils.processors.bagel",
"CheersProcessor": "vllm.transformers_utils.processors.cheers",
"CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr",
"Cosmos3EdgeProcessor": "vllm.transformers_utils.processors.cosmos3_edge",
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
"FireRedLIDProcessor": "vllm.transformers_utils.processors.fireredlid",
@@ -0,0 +1,356 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import torch
from torchvision.transforms import InterpolationMode
from transformers import AutoTokenizer
from transformers.feature_extraction_utils import BatchFeature
from transformers.image_utils import ChannelDimension, PILImageResampling, SizeDict
from transformers.models.qwen3_vl.processing_qwen3_vl import Qwen3VLProcessor
from transformers.models.qwen3_vl.video_processing_qwen3_vl import (
Qwen3VLVideoProcessor,
Qwen3VLVideoProcessorInitKwargs,
get_image_size,
smart_resize,
)
from transformers.models.siglip2.image_processing_siglip2 import (
Siglip2ImageProcessor,
convert_image_to_patches,
)
from transformers.processing_utils import Unpack
from transformers.utils import TensorType
from transformers.video_utils import group_videos_by_shape, reorder_videos
def round_by_factor(number: int, factor: int) -> int:
"""Returns the closest integer to 'number' that is divisible by 'factor'."""
return round(number / factor) * factor
def ceil_by_factor(number: int, factor: int) -> int:
"""Return the smallest multiple of ``factor`` greater than ``number``."""
return math.ceil(number / factor) * factor
def floor_by_factor(number: int, factor: int) -> int:
"""Return the largest multiple of ``factor`` less than ``number``."""
return math.floor(number / factor) * factor
class Cosmos3EdgeImagesKwargs( # type: ignore[call-arg]
Siglip2ImageProcessor.valid_kwargs,
total=False,
):
# global setting for all images, can be overridden by per-image kwargs
max_pixels: int | None
min_pixels: int | None
per_image_kwargs: list[dict] | None
class Cosmos3EdgeImageProcessor(Siglip2ImageProcessor):
resample = PILImageResampling.BICUBIC
valid_kwargs = Cosmos3EdgeImagesKwargs
def __init__(self, **kwargs: Unpack[Cosmos3EdgeImagesKwargs]):
super().__init__(**kwargs)
def _resize_image(
self,
image: torch.Tensor,
max_ratio=200,
resample: PILImageResampling | InterpolationMode | int | None = None,
**kwargs,
) -> torch.Tensor:
"""
Rescales the image so that the following conditions are met:
1. Both dimensions (height and width) are divisible by 'factor'.
2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
3. The aspect ratio of the image is maintained as closely as possible.
"""
image_min_pixels = kwargs.get("min_pixels") or self.size.get(
"shortest_edge", None
)
image_max_pixels = kwargs.get("max_pixels") or self.size.get(
"longest_edge", None
)
assert image_min_pixels is not None and image_max_pixels is not None, (
"When do_resize is True, min_pixels and max_pixels must be provided."
)
assert image_max_pixels >= image_min_pixels, (
"The max_pixels of image must be greater than or equal to min_pixels."
)
_, height, width = image.shape
aspect_ratio = max(height, width) / min(height, width)
if aspect_ratio > max_ratio:
raise ValueError(
f"absolute aspect ratio must be smaller than {max_ratio}, "
f"got {aspect_ratio}"
)
factor = self.merge_size * self.patch_size
h_bar = max(factor, round_by_factor(height, factor))
w_bar = max(factor, round_by_factor(width, factor))
if h_bar * w_bar > image_max_pixels:
beta = math.sqrt((height * width) / image_max_pixels)
h_bar = floor_by_factor(height / beta, factor)
w_bar = floor_by_factor(width / beta, factor)
elif h_bar * w_bar < image_min_pixels:
beta = math.sqrt(image_min_pixels / (height * width))
h_bar = ceil_by_factor(height * beta, factor)
w_bar = ceil_by_factor(width * beta, factor)
image = self.resize(
image,
size=SizeDict(height=h_bar, width=w_bar),
resample=resample,
)
return image
def _preprocess(
self,
images: list["torch.Tensor"],
do_resize: bool,
patch_size: int,
max_num_patches: int,
resample: PILImageResampling | InterpolationMode | int | None,
do_rescale: bool,
rescale_factor: float,
do_normalize: bool,
image_mean: float | list[float] | None,
image_std: float | list[float] | None,
return_tensors: str | TensorType | None,
**kwargs,
) -> BatchFeature:
per_image_kwargs = kwargs.pop("per_image_kwargs", None)
pixel_values = []
spatial_shapes = []
for idx, image in enumerate(images):
# merge global kwargs with per-image overrides (if provided)
image_kwargs = kwargs.copy()
if (
per_image_kwargs is not None
and idx < len(per_image_kwargs)
and per_image_kwargs[idx]
):
image_kwargs.update(per_image_kwargs[idx])
if do_resize:
image = self._resize_image(
image,
max_ratio=200,
resample=resample,
**image_kwargs,
)
image = self.rescale_and_normalize(
image, do_rescale, rescale_factor, do_normalize, image_mean, image_std
)
# (channels, height, width) -> (patches, patch_size**2 * channels)
patches = convert_image_to_patches(image, patch_size)
num_patches_height = image.shape[1] // patch_size
num_patches_width = image.shape[2] // patch_size
spatial_shapes.append((num_patches_height, num_patches_width))
pixel_values.append(patches)
pixel_values = torch.cat(pixel_values, dim=0)
spatial_shapes = torch.tensor(spatial_shapes)
t_dim = torch.ones(
(spatial_shapes.shape[0], 1),
dtype=spatial_shapes.dtype,
device=spatial_shapes.device,
)
image_grid_thw = torch.cat([t_dim, spatial_shapes], dim=1)
batch_feature = BatchFeature(
data={
"pixel_values": pixel_values,
"image_grid_thw": image_grid_thw,
},
tensor_type=return_tensors,
)
return batch_feature
class Cosmos3EdgeVideoProcessor(Qwen3VLVideoProcessor):
def __init__(self, **kwargs: Unpack[Qwen3VLVideoProcessorInitKwargs]):
super().__init__(**kwargs)
def _preprocess(
self,
videos: list[torch.Tensor],
do_convert_rgb: bool = True,
do_resize: bool = True,
size: SizeDict | None = None,
resample: PILImageResampling
| InterpolationMode
| int
| None = PILImageResampling.BICUBIC,
do_rescale: bool = True,
rescale_factor: float = 1 / 255.0,
do_normalize: bool = True,
image_mean: float | list[float] | None = None,
image_std: float | list[float] | None = None,
patch_size: int | None = None,
return_tensors: str | TensorType | None = None,
**kwargs,
):
assert patch_size is not None
merge_size = self.merge_size
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
resized_videos_grouped = {}
for shape, stacked_videos in grouped_videos.items():
B, T, C, H, W = stacked_videos.shape
num_frames, height, width = T, H, W
if do_resize:
assert size is not None
# 1. Both dimensions (height and width) are divisible by 'factor'.
# 2. Total pixels are within the configured min/max range.
resized_height, resized_width = smart_resize(
num_frames=num_frames,
height=height,
width=width,
temporal_factor=1,
factor=patch_size * merge_size,
min_pixels=size.shortest_edge,
max_pixels=size.longest_edge,
)
stacked_videos = stacked_videos.view(B * T, C, H, W)
stacked_videos = self.resize(
stacked_videos,
size=SizeDict(height=resized_height, width=resized_width),
resample=resample,
)
stacked_videos = stacked_videos.view(
B, T, C, resized_height, resized_width
)
resized_videos_grouped[shape] = stacked_videos
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
# Group videos by size for further processing
# Resize may return videos with different sizes.
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
processed_videos_grouped = {}
processed_grids = {}
for shape, stacked_videos in grouped_videos.items():
resized_height, resized_width = get_image_size(
stacked_videos[0], channel_dim=ChannelDimension.FIRST
)
# Fused rescale and normalize
stacked_videos = self.rescale_and_normalize(
stacked_videos,
do_rescale,
rescale_factor,
do_normalize,
image_mean,
image_std,
)
patches = stacked_videos
batch_size, grid_t, channel = patches.shape[:3]
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
# [1, 16, 3, 320, 608] -> [1, 16, 3, 20, 16, 38, 16]
patches = patches.view(
batch_size, # 0
grid_t, # 1
channel, # 2
grid_h, # 3
patch_size, # 4
grid_w, # 5
patch_size, # 6
)
# [1, 16, 3, 20, 16, 38, 16] -> [1, 16, 20, 38, 16, 16, 3]
# Result: [batch, grid_t, grid_h, grid_w, patch_h, patch_w, channel]
patches = patches.permute(0, 1, 3, 5, 4, 6, 2)
# [1, 16, 20, 38, 16, 16, 3] -> [1, 5120, 768]
flatten_patches = patches.reshape(
batch_size,
grid_t * grid_h * grid_w,
patch_size * patch_size * channel,
)
processed_videos_grouped[shape] = flatten_patches
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
processed_videos = reorder_videos(
processed_videos_grouped, grouped_videos_index
)
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
pixel_values_videos = torch.cat(processed_videos, dim=0)
video_grid_thw = torch.tensor(processed_grids)
data = {
"pixel_values_videos": pixel_values_videos,
"video_grid_thw": video_grid_thw,
}
return BatchFeature(data=data, tensor_type=return_tensors)
class Cosmos3EdgeProcessor(Qwen3VLProcessor):
"""Construct a processor for Cosmos3 Edge multimodal inputs.
This processor combines :class:`Cosmos3EdgeImageProcessor`,
:class:`Cosmos3EdgeVideoProcessor`, and a tokenizer into a single
interface. It inherits the text, image, and video input handling and the
decoding methods from :class:`Qwen3VLProcessor`, while using the Cosmos3
Edge-specific image and video preprocessing implementations.
Args:
image_processor: A :class:`Cosmos3EdgeImageProcessor` used to resize,
normalize, and patch image inputs.
tokenizer: The tokenizer used to encode and decode text.
video_processor: A :class:`Cosmos3EdgeVideoProcessor` used to resize,
normalize, and patch video inputs.
chat_template: An optional Jinja template used to render chat messages
into a tokenizable string.
"""
attributes = ["image_processor", "tokenizer", "video_processor"]
# Transformers' auto loaders cannot resolve these vLLM-native image and
# video processors without remote code or global auto-class registration.
@classmethod
def _get_arguments_from_pretrained(
cls,
pretrained_model_name_or_path,
processor_dict=None,
**kwargs,
):
component_kwargs = dict(kwargs)
component_kwargs.pop("trust_remote_code", None)
image_processor = Cosmos3EdgeImageProcessor.from_pretrained(
pretrained_model_name_or_path,
**component_kwargs,
)
tokenizer = AutoTokenizer.from_pretrained(
pretrained_model_name_or_path,
trust_remote_code=False,
**component_kwargs,
)
video_processor = Cosmos3EdgeVideoProcessor.from_pretrained(
pretrained_model_name_or_path,
**component_kwargs,
)
return [
image_processor,
tokenizer,
video_processor,
]
__all__ = [
"Cosmos3EdgeProcessor",
]