Add Cosmos3 Edge Reasoner model (#48291)

Signed-off-by: Bartosz Stefaniak <bstefaniak@nvidia.com>
Co-authored-by: Bartosz Stefaniak <bstefaniak@nvidia.com>
This commit is contained in:
adsridhar
2026-07-14 08:14:50 +00:00
committed by GitHub
co-authored by Bartosz Stefaniak
parent c9a788eedc
commit 894ebb27f5
13 changed files with 1474 additions and 27 deletions
+1
View File
@@ -81,6 +81,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
ops_colqwen3="OpsColQwen3Config",
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
cosmos3_omni="Cosmos3Config",
cosmos3_edge="Cosmos3EdgeConfig",
diffusion_gemma="DiffusionGemmaConfig",
deepseek_vl_v2="DeepseekVLV2Config",
deepseek_v32="DeepseekV3Config",
@@ -27,6 +27,10 @@ _CLASS_TO_MODULE: dict[str, str] = {
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
"Cosmos3Config": "vllm.transformers_utils.configs.cosmos3",
"Cosmos3EdgeConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeProjectorConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge",
"DiffusionGemmaConfig": "vllm.transformers_utils.configs.diffusion_gemma",
"DiffusionGemmaTextConfig": "vllm.transformers_utils.configs.diffusion_gemma",
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
@@ -107,6 +111,10 @@ __all__ = [
"OpsColQwen3Config",
"Qwen3VLNemotronEmbedConfig",
"Cosmos3Config",
"Cosmos3EdgeConfig",
"Cosmos3EdgeProjectorConfig",
"Cosmos3EdgeTextConfig",
"Cosmos3EdgeVisionConfig",
"DiffusionGemmaConfig",
"DiffusionGemmaTextConfig",
"DeepseekVLV2Config",
@@ -0,0 +1,146 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from transformers.configuration_utils import PretrainedConfig
from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig
from .nemotron_h import NemotronHConfig
def _normalize_mrope_parameters(config: NemotronHConfig) -> None:
"""Normalize the checkpoint's mRoPE configuration for vLLM."""
rope_parameters = dict(getattr(config, "rope_parameters", None) or {})
mrope_section = rope_parameters.get("mrope_section")
if mrope_section is None:
return
rope_parameters["mrope_section"] = list(mrope_section)
rope_parameters.setdefault("mrope_interleaved", True)
config.rope_parameters = rope_parameters
class Cosmos3EdgeTextConfig(NemotronHConfig):
"""Dense Cosmos3 Edge text config backed by Nemotron-H layers."""
model_type = "cosmos3_edge_text"
has_no_defaults_at_init = True
ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"}
def __init__(
self,
num_hidden_layers: int = 28,
hidden_act: str = "relu2",
rms_norm_eps: float = 1e-5,
**kwargs,
) -> None:
super().__init__(
num_hidden_layers=2 * num_hidden_layers,
hybrid_override_pattern="*-" * num_hidden_layers,
mlp_hidden_act=hidden_act,
layer_norm_epsilon=rms_norm_eps,
**kwargs,
)
def to_dict(self) -> dict:
config_dict = super().to_dict()
config_dict["num_hidden_layers"] = self.num_hidden_layers // 2
config_dict["hidden_act"] = config_dict.pop("mlp_hidden_act")
config_dict["rms_norm_eps"] = config_dict.pop("layer_norm_epsilon")
config_dict.pop("hybrid_override_pattern", None)
return config_dict
class Cosmos3EdgeProjectorConfig(PretrainedConfig):
model_type = "cosmos3_edge_projector"
def __init__(
self,
input_hidden_size: int = 1024,
use_postshuffle_norm: bool = False,
spatial_merge_size: int = 2,
merger_intermediate_size: int = 11520,
out_hidden_size: int = 1024,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.input_hidden_size = input_hidden_size
self.use_postshuffle_norm = use_postshuffle_norm
self.spatial_merge_size = spatial_merge_size
self.merger_intermediate_size = merger_intermediate_size
self.out_hidden_size = out_hidden_size
class Cosmos3EdgeVisionConfig(Siglip2VisionConfig):
model_type = "cosmos3_edge_vision"
class Cosmos3EdgeConfig(PretrainedConfig):
model_type = "cosmos3_edge"
keys_to_ignore_at_inference = ["past_key_values"]
has_no_defaults_at_init = True
sub_configs = {
"vision_config": Cosmos3EdgeVisionConfig,
"projector_config": Cosmos3EdgeProjectorConfig,
"text_config": Cosmos3EdgeTextConfig,
}
def __init__(
self,
text_config: Cosmos3EdgeTextConfig | dict | None = None,
vision_config: Cosmos3EdgeVisionConfig | dict | None = None,
projector_config: Cosmos3EdgeProjectorConfig | dict | None = None,
image_token_id: int = 19,
video_token_id: int = 18,
vision_start_token_id: int = 20,
vision_end_token_id: int = 21,
tie_word_embeddings: bool = False,
**kwargs,
) -> None:
if text_config is None:
text_config = {}
if vision_config is None:
vision_config = {}
if projector_config is None:
projector_config = {}
self.text_config = (
Cosmos3EdgeTextConfig(**text_config)
if isinstance(text_config, dict)
else text_config
)
_normalize_mrope_parameters(self.text_config)
self.vision_config = (
Cosmos3EdgeVisionConfig(**vision_config)
if isinstance(vision_config, dict)
else vision_config
)
self.projector_config = (
Cosmos3EdgeProjectorConfig(**projector_config)
if isinstance(projector_config, dict)
else projector_config
)
# Qwen3-VL processing and M-RoPE read these attributes from the
# vision config, while this checkpoint stores them in projector config.
self.vision_config.spatial_merge_size = self.projector_config.spatial_merge_size
self.vision_config.temporal_patch_size = 1
self.vision_config.out_hidden_size = self.projector_config.out_hidden_size
self.image_token_id = image_token_id
self.video_token_id = video_token_id
self.vision_start_token_id = vision_start_token_id
self.vision_end_token_id = vision_end_token_id
super().__init__(
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
__all__ = [
"Cosmos3EdgeConfig",
"Cosmos3EdgeProjectorConfig",
"Cosmos3EdgeTextConfig",
"Cosmos3EdgeVisionConfig",
]
@@ -14,6 +14,7 @@ __all__ = [
"BagelProcessor",
"CheersProcessor",
"CohereASRProcessor",
"Cosmos3EdgeProcessor",
"DeepseekVLV2Processor",
"FireRedASR2Processor",
"FireRedLIDProcessor",
@@ -49,6 +50,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"BagelProcessor": "vllm.transformers_utils.processors.bagel",
"CheersProcessor": "vllm.transformers_utils.processors.cheers",
"CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr",
"Cosmos3EdgeProcessor": "vllm.transformers_utils.processors.cosmos3_edge",
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
"FireRedLIDProcessor": "vllm.transformers_utils.processors.fireredlid",
@@ -0,0 +1,356 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import torch
from torchvision.transforms import InterpolationMode
from transformers import AutoTokenizer
from transformers.feature_extraction_utils import BatchFeature
from transformers.image_utils import ChannelDimension, PILImageResampling, SizeDict
from transformers.models.qwen3_vl.processing_qwen3_vl import Qwen3VLProcessor
from transformers.models.qwen3_vl.video_processing_qwen3_vl import (
Qwen3VLVideoProcessor,
Qwen3VLVideoProcessorInitKwargs,
get_image_size,
smart_resize,
)
from transformers.models.siglip2.image_processing_siglip2 import (
Siglip2ImageProcessor,
convert_image_to_patches,
)
from transformers.processing_utils import Unpack
from transformers.utils import TensorType
from transformers.video_utils import group_videos_by_shape, reorder_videos
def round_by_factor(number: int, factor: int) -> int:
"""Returns the closest integer to 'number' that is divisible by 'factor'."""
return round(number / factor) * factor
def ceil_by_factor(number: int, factor: int) -> int:
"""Return the smallest multiple of ``factor`` greater than ``number``."""
return math.ceil(number / factor) * factor
def floor_by_factor(number: int, factor: int) -> int:
"""Return the largest multiple of ``factor`` less than ``number``."""
return math.floor(number / factor) * factor
class Cosmos3EdgeImagesKwargs( # type: ignore[call-arg]
Siglip2ImageProcessor.valid_kwargs,
total=False,
):
# global setting for all images, can be overridden by per-image kwargs
max_pixels: int | None
min_pixels: int | None
per_image_kwargs: list[dict] | None
class Cosmos3EdgeImageProcessor(Siglip2ImageProcessor):
resample = PILImageResampling.BICUBIC
valid_kwargs = Cosmos3EdgeImagesKwargs
def __init__(self, **kwargs: Unpack[Cosmos3EdgeImagesKwargs]):
super().__init__(**kwargs)
def _resize_image(
self,
image: torch.Tensor,
max_ratio=200,
resample: PILImageResampling | InterpolationMode | int | None = None,
**kwargs,
) -> torch.Tensor:
"""
Rescales the image so that the following conditions are met:
1. Both dimensions (height and width) are divisible by 'factor'.
2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
3. The aspect ratio of the image is maintained as closely as possible.
"""
image_min_pixels = kwargs.get("min_pixels") or self.size.get(
"shortest_edge", None
)
image_max_pixels = kwargs.get("max_pixels") or self.size.get(
"longest_edge", None
)
assert image_min_pixels is not None and image_max_pixels is not None, (
"When do_resize is True, min_pixels and max_pixels must be provided."
)
assert image_max_pixels >= image_min_pixels, (
"The max_pixels of image must be greater than or equal to min_pixels."
)
_, height, width = image.shape
aspect_ratio = max(height, width) / min(height, width)
if aspect_ratio > max_ratio:
raise ValueError(
f"absolute aspect ratio must be smaller than {max_ratio}, "
f"got {aspect_ratio}"
)
factor = self.merge_size * self.patch_size
h_bar = max(factor, round_by_factor(height, factor))
w_bar = max(factor, round_by_factor(width, factor))
if h_bar * w_bar > image_max_pixels:
beta = math.sqrt((height * width) / image_max_pixels)
h_bar = floor_by_factor(height / beta, factor)
w_bar = floor_by_factor(width / beta, factor)
elif h_bar * w_bar < image_min_pixels:
beta = math.sqrt(image_min_pixels / (height * width))
h_bar = ceil_by_factor(height * beta, factor)
w_bar = ceil_by_factor(width * beta, factor)
image = self.resize(
image,
size=SizeDict(height=h_bar, width=w_bar),
resample=resample,
)
return image
def _preprocess(
self,
images: list["torch.Tensor"],
do_resize: bool,
patch_size: int,
max_num_patches: int,
resample: PILImageResampling | InterpolationMode | int | None,
do_rescale: bool,
rescale_factor: float,
do_normalize: bool,
image_mean: float | list[float] | None,
image_std: float | list[float] | None,
return_tensors: str | TensorType | None,
**kwargs,
) -> BatchFeature:
per_image_kwargs = kwargs.pop("per_image_kwargs", None)
pixel_values = []
spatial_shapes = []
for idx, image in enumerate(images):
# merge global kwargs with per-image overrides (if provided)
image_kwargs = kwargs.copy()
if (
per_image_kwargs is not None
and idx < len(per_image_kwargs)
and per_image_kwargs[idx]
):
image_kwargs.update(per_image_kwargs[idx])
if do_resize:
image = self._resize_image(
image,
max_ratio=200,
resample=resample,
**image_kwargs,
)
image = self.rescale_and_normalize(
image, do_rescale, rescale_factor, do_normalize, image_mean, image_std
)
# (channels, height, width) -> (patches, patch_size**2 * channels)
patches = convert_image_to_patches(image, patch_size)
num_patches_height = image.shape[1] // patch_size
num_patches_width = image.shape[2] // patch_size
spatial_shapes.append((num_patches_height, num_patches_width))
pixel_values.append(patches)
pixel_values = torch.cat(pixel_values, dim=0)
spatial_shapes = torch.tensor(spatial_shapes)
t_dim = torch.ones(
(spatial_shapes.shape[0], 1),
dtype=spatial_shapes.dtype,
device=spatial_shapes.device,
)
image_grid_thw = torch.cat([t_dim, spatial_shapes], dim=1)
batch_feature = BatchFeature(
data={
"pixel_values": pixel_values,
"image_grid_thw": image_grid_thw,
},
tensor_type=return_tensors,
)
return batch_feature
class Cosmos3EdgeVideoProcessor(Qwen3VLVideoProcessor):
def __init__(self, **kwargs: Unpack[Qwen3VLVideoProcessorInitKwargs]):
super().__init__(**kwargs)
def _preprocess(
self,
videos: list[torch.Tensor],
do_convert_rgb: bool = True,
do_resize: bool = True,
size: SizeDict | None = None,
resample: PILImageResampling
| InterpolationMode
| int
| None = PILImageResampling.BICUBIC,
do_rescale: bool = True,
rescale_factor: float = 1 / 255.0,
do_normalize: bool = True,
image_mean: float | list[float] | None = None,
image_std: float | list[float] | None = None,
patch_size: int | None = None,
return_tensors: str | TensorType | None = None,
**kwargs,
):
assert patch_size is not None
merge_size = self.merge_size
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
resized_videos_grouped = {}
for shape, stacked_videos in grouped_videos.items():
B, T, C, H, W = stacked_videos.shape
num_frames, height, width = T, H, W
if do_resize:
assert size is not None
# 1. Both dimensions (height and width) are divisible by 'factor'.
# 2. Total pixels are within the configured min/max range.
resized_height, resized_width = smart_resize(
num_frames=num_frames,
height=height,
width=width,
temporal_factor=1,
factor=patch_size * merge_size,
min_pixels=size.shortest_edge,
max_pixels=size.longest_edge,
)
stacked_videos = stacked_videos.view(B * T, C, H, W)
stacked_videos = self.resize(
stacked_videos,
size=SizeDict(height=resized_height, width=resized_width),
resample=resample,
)
stacked_videos = stacked_videos.view(
B, T, C, resized_height, resized_width
)
resized_videos_grouped[shape] = stacked_videos
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
# Group videos by size for further processing
# Resize may return videos with different sizes.
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
processed_videos_grouped = {}
processed_grids = {}
for shape, stacked_videos in grouped_videos.items():
resized_height, resized_width = get_image_size(
stacked_videos[0], channel_dim=ChannelDimension.FIRST
)
# Fused rescale and normalize
stacked_videos = self.rescale_and_normalize(
stacked_videos,
do_rescale,
rescale_factor,
do_normalize,
image_mean,
image_std,
)
patches = stacked_videos
batch_size, grid_t, channel = patches.shape[:3]
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
# [1, 16, 3, 320, 608] -> [1, 16, 3, 20, 16, 38, 16]
patches = patches.view(
batch_size, # 0
grid_t, # 1
channel, # 2
grid_h, # 3
patch_size, # 4
grid_w, # 5
patch_size, # 6
)
# [1, 16, 3, 20, 16, 38, 16] -> [1, 16, 20, 38, 16, 16, 3]
# Result: [batch, grid_t, grid_h, grid_w, patch_h, patch_w, channel]
patches = patches.permute(0, 1, 3, 5, 4, 6, 2)
# [1, 16, 20, 38, 16, 16, 3] -> [1, 5120, 768]
flatten_patches = patches.reshape(
batch_size,
grid_t * grid_h * grid_w,
patch_size * patch_size * channel,
)
processed_videos_grouped[shape] = flatten_patches
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
processed_videos = reorder_videos(
processed_videos_grouped, grouped_videos_index
)
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
pixel_values_videos = torch.cat(processed_videos, dim=0)
video_grid_thw = torch.tensor(processed_grids)
data = {
"pixel_values_videos": pixel_values_videos,
"video_grid_thw": video_grid_thw,
}
return BatchFeature(data=data, tensor_type=return_tensors)
class Cosmos3EdgeProcessor(Qwen3VLProcessor):
"""Construct a processor for Cosmos3 Edge multimodal inputs.
This processor combines :class:`Cosmos3EdgeImageProcessor`,
:class:`Cosmos3EdgeVideoProcessor`, and a tokenizer into a single
interface. It inherits the text, image, and video input handling and the
decoding methods from :class:`Qwen3VLProcessor`, while using the Cosmos3
Edge-specific image and video preprocessing implementations.
Args:
image_processor: A :class:`Cosmos3EdgeImageProcessor` used to resize,
normalize, and patch image inputs.
tokenizer: The tokenizer used to encode and decode text.
video_processor: A :class:`Cosmos3EdgeVideoProcessor` used to resize,
normalize, and patch video inputs.
chat_template: An optional Jinja template used to render chat messages
into a tokenizable string.
"""
attributes = ["image_processor", "tokenizer", "video_processor"]
# Transformers' auto loaders cannot resolve these vLLM-native image and
# video processors without remote code or global auto-class registration.
@classmethod
def _get_arguments_from_pretrained(
cls,
pretrained_model_name_or_path,
processor_dict=None,
**kwargs,
):
component_kwargs = dict(kwargs)
component_kwargs.pop("trust_remote_code", None)
image_processor = Cosmos3EdgeImageProcessor.from_pretrained(
pretrained_model_name_or_path,
**component_kwargs,
)
tokenizer = AutoTokenizer.from_pretrained(
pretrained_model_name_or_path,
trust_remote_code=False,
**component_kwargs,
)
video_processor = Cosmos3EdgeVideoProcessor.from_pretrained(
pretrained_model_name_or_path,
**component_kwargs,
)
return [
image_processor,
tokenizer,
video_processor,
]
__all__ = [
"Cosmos3EdgeProcessor",
]