Add Cosmos3 Edge Reasoner model (#48291)
Signed-off-by: Bartosz Stefaniak <bstefaniak@nvidia.com> Co-authored-by: Bartosz Stefaniak <bstefaniak@nvidia.com>
This commit is contained in:
co-authored by
Bartosz Stefaniak
parent
c9a788eedc
commit
894ebb27f5
@@ -514,6 +514,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| `CheersForConditionalGeneration` | Cheers | T + I | `ai9stars/Cheers` | | ✅︎ |
|
||||
| `Cohere2VisionForConditionalGeneration` | Command A Vision, Command-A+ | T + I<sup>+</sup> | `CohereLabs/command-a-vision-07-2025`, `CohereLabs/command-a-plus-05-2026`, etc. | | ✅︎ |
|
||||
| `Cosmos3ForConditionalGeneration` | Cosmos3 (understanding tower) | T + I<sup>E+</sup> + V<sup>E+</sup> | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | | ✅︎ |
|
||||
| `Cosmos3EdgeForConditionalGeneration` | Cosmos3-Edge (understanding tower) | T + I<sup>E+</sup> + V<sup>E+</sup> | `nvidia/Cosmos3-Edge` | | ✅︎ |
|
||||
| `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
|
||||
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
|
||||
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
MODEL_ID = "nvidia/Cosmos3-Edge"
|
||||
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
LOCAL_MODEL_PATH = os.getenv("COSMOS3_EDGE_MODEL_PATH")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def processor():
|
||||
if LOCAL_MODEL_PATH:
|
||||
model_config = ModelConfig(
|
||||
LOCAL_MODEL_PATH,
|
||||
tokenizer=LOCAL_MODEL_PATH,
|
||||
max_model_len=4096,
|
||||
limit_mm_per_prompt={"image": 2, "video": 1},
|
||||
)
|
||||
else:
|
||||
ctx = build_model_context(
|
||||
MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 2, "video": 1},
|
||||
)
|
||||
model_config = ctx.model_config
|
||||
|
||||
return MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
|
||||
def _assert_image_outputs(processor, processed, num_images: int) -> None:
|
||||
mm_data = processed["mm_kwargs"].get_data()
|
||||
grid_thw = mm_data["image_grid_thw"]
|
||||
pixel_values = mm_data["pixel_values"]
|
||||
|
||||
assert grid_thw.shape == (num_images, 3)
|
||||
assert pixel_values.shape[0] == int(grid_thw.prod(dim=-1).sum())
|
||||
|
||||
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
|
||||
expected_tokens = (grid_thw.prod(dim=-1) // merge_size**2).tolist()
|
||||
image_placeholders = processed["mm_placeholders"]["image"]
|
||||
|
||||
assert len(image_placeholders) == num_images
|
||||
assert [placeholder.length for placeholder in image_placeholders] == (
|
||||
expected_tokens
|
||||
)
|
||||
|
||||
image_token_id = processor.info.get_hf_processor().image_token_id
|
||||
assert processed["prompt_token_ids"].count(image_token_id) == sum(expected_tokens)
|
||||
|
||||
|
||||
def _assert_video_outputs(processor, processed) -> None:
|
||||
mm_data = processed["mm_kwargs"].get_data()
|
||||
grid_thw = mm_data["video_grid_thw"]
|
||||
pixel_values = mm_data["pixel_values_videos"]
|
||||
|
||||
assert grid_thw.shape == (1, 3)
|
||||
assert pixel_values.shape[0] == int(grid_thw.prod())
|
||||
assert len(processed["mm_placeholders"]["video"]) == 1
|
||||
|
||||
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
|
||||
expected_tokens = int(grid_thw.prod()) // merge_size**2
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
assert processed["prompt_token_ids"].count(video_token_id) == expected_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_images", [1, 2])
|
||||
def test_process_images(
|
||||
processor,
|
||||
image_assets: ImageTestAssets,
|
||||
num_images: int,
|
||||
) -> None:
|
||||
images = [asset.pil_image for asset in image_assets[:num_images]]
|
||||
processed = processor(
|
||||
IMAGE_PLACEHOLDER * num_images,
|
||||
mm_items=processor.info.parse_mm_data({"image": images}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
_assert_image_outputs(processor, processed, num_images)
|
||||
|
||||
|
||||
def test_process_video(processor) -> None:
|
||||
video_asset = VideoAsset(name="baby_reading", num_frames=8)
|
||||
video = (video_asset.np_ndarrays, video_asset.metadata)
|
||||
processed = processor(
|
||||
VIDEO_PLACEHOLDER,
|
||||
mm_items=processor.info.parse_mm_data({"video": [video]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
_assert_video_outputs(processor, processed)
|
||||
@@ -74,6 +74,77 @@ def test_cosmos3_new_checkpoint_weights_mapper():
|
||||
)
|
||||
|
||||
|
||||
def test_cosmos3_edge_checkpoint_weights_mapper():
|
||||
from vllm.model_executor.models.cosmos3_edge import (
|
||||
Cosmos3EdgeForConditionalGeneration,
|
||||
)
|
||||
|
||||
mapper = Cosmos3EdgeForConditionalGeneration.hf_to_vllm_mapper
|
||||
|
||||
assert mapper.apply_list(
|
||||
[
|
||||
"embed_tokens.weight",
|
||||
"norm.weight",
|
||||
"layers.0.input_layernorm.weight",
|
||||
"layers.0.self_attn.to_q.weight",
|
||||
"layers.0.self_attn.to_k.weight",
|
||||
"layers.0.self_attn.to_v.weight",
|
||||
"layers.0.self_attn.to_out.weight",
|
||||
"layers.0.post_attention_layernorm.weight",
|
||||
"layers.0.mlp.up_proj.weight",
|
||||
"layers.0.mlp.down_proj.weight",
|
||||
"layers.27.input_layernorm.weight",
|
||||
"layers.27.self_attn.to_q.weight",
|
||||
"layers.27.post_attention_layernorm.weight",
|
||||
"layers.27.mlp.down_proj.weight",
|
||||
"model.visual.embeddings.patch_embedding.weight",
|
||||
"model.visual.encoder.layers.0.self_attn.q_proj.weight",
|
||||
"model.projector.linear_fc1.weight",
|
||||
"lm_head.weight",
|
||||
]
|
||||
) == [
|
||||
"language_model.model.embed_tokens.weight",
|
||||
"language_model.model.norm_f.weight",
|
||||
"language_model.model.layers.0.norm.weight",
|
||||
"language_model.model.layers.0.mixer.qkv_proj.weight",
|
||||
"language_model.model.layers.0.mixer.qkv_proj.weight",
|
||||
"language_model.model.layers.0.mixer.qkv_proj.weight",
|
||||
"language_model.model.layers.0.mixer.o_proj.weight",
|
||||
"language_model.model.layers.1.norm.weight",
|
||||
"language_model.model.layers.1.mixer.up_proj.weight",
|
||||
"language_model.model.layers.1.mixer.down_proj.weight",
|
||||
"language_model.model.layers.54.norm.weight",
|
||||
"language_model.model.layers.54.mixer.qkv_proj.weight",
|
||||
"language_model.model.layers.55.norm.weight",
|
||||
"language_model.model.layers.55.mixer.down_proj.weight",
|
||||
"visual.encoder.embeddings.patch_embedding.weight",
|
||||
"visual.encoder.encoder.layers.0.self_attn.qkv_proj.weight",
|
||||
"visual.projector.linear_fc1.weight",
|
||||
"language_model.lm_head.weight",
|
||||
]
|
||||
|
||||
assert (
|
||||
mapper.apply_list(
|
||||
[
|
||||
"layers.0.self_attn.add_q_proj.weight",
|
||||
"layers.0.self_attn.add_k_proj.weight",
|
||||
"layers.0.self_attn.add_v_proj.weight",
|
||||
"layers.0.self_attn.to_add_out.weight",
|
||||
"layers.0.self_attn.norm_added_q.weight",
|
||||
"layers.0.self_attn.norm_added_k.weight",
|
||||
"layers.0.self_attn.q_proj_moe_gen.weight",
|
||||
"layers.0.mlp_moe_gen.up_proj.weight",
|
||||
"norm_moe_gen.weight",
|
||||
"proj_in.weight",
|
||||
"proj_out.weight",
|
||||
"audio_modality_embed",
|
||||
"action_modality_embed",
|
||||
]
|
||||
)
|
||||
== []
|
||||
)
|
||||
|
||||
|
||||
def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Create weights from safetensors checkpoint metadata"""
|
||||
metadata = try_get_safetensors_metadata(repo)
|
||||
|
||||
@@ -811,6 +811,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/Cosmos3-Edge",
|
||||
max_model_len=4096,
|
||||
is_available_online=False,
|
||||
),
|
||||
"DeepseekVLV2ForCausalLM": _HfExamplesInfo(
|
||||
"deepseek-ai/deepseek-vl2-tiny",
|
||||
extras={"fork": "Isotr0py/deepseek-vl2-tiny"},
|
||||
|
||||
@@ -3,8 +3,6 @@
|
||||
|
||||
import regex
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
@@ -52,22 +50,7 @@ class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
},
|
||||
)
|
||||
|
||||
allow_patterns_overrides = ["transformer/*.safetensors"]
|
||||
|
||||
"""
|
||||
Cosmos3 checkpoint separates transformer weights and vision_encoder weights
|
||||
into separate directories, as it's in diffusers checkpoint format.
|
||||
Using secondary_weights here to load all necessary weights for
|
||||
the Reasoner-only part.
|
||||
"""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
self.secondary_weights = [
|
||||
DefaultModelLoader.Source(
|
||||
model_or_path=vllm_config.model_config.model,
|
||||
revision=vllm_config.model_config.revision,
|
||||
prefix="",
|
||||
allow_patterns_overrides=["vision_encoder/*.safetensors"],
|
||||
),
|
||||
]
|
||||
# Cosmos3 unified Diffusers checkpoints store reasoner weights across
|
||||
# transformer/ and vision_encoder/. Match both while excluding VAE and
|
||||
# sound_tokenizer weights; the mapper drops generation-only tensors.
|
||||
allow_patterns_overrides = ["[tv]*er/*.safetensors"]
|
||||
|
||||
@@ -0,0 +1,751 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import ProcessorMixin
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from .lfm2_siglip2 import Siglip2VisionTransformer
|
||||
from .nemotron_h import (
|
||||
NemotronHAttention,
|
||||
NemotronHMLPDecoderLayer,
|
||||
NemotronHModel,
|
||||
)
|
||||
from .qwen2_5_vl import (
|
||||
Qwen2_5_VLImageEmbeddingInputs,
|
||||
Qwen2_5_VLImageInputs,
|
||||
Qwen2_5_VLImagePixelInputs,
|
||||
Qwen2_5_VLVideoEmbeddingInputs,
|
||||
Qwen2_5_VLVideoInputs,
|
||||
Qwen2_5_VLVideoPixelInputs,
|
||||
)
|
||||
from .qwen3_vl import (
|
||||
Qwen3VLDummyInputsBuilder,
|
||||
Qwen3VLForConditionalGeneration,
|
||||
Qwen3VLMultiModalProcessor,
|
||||
Qwen3VLProcessingInfo,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .vision import (
|
||||
is_vit_use_data_parallel,
|
||||
run_dp_sharded_mrope_vision_model,
|
||||
)
|
||||
|
||||
|
||||
class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer):
|
||||
"""Adapts Cosmos (T, H, W) metadata to vLLM packed SigLIP2."""
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return self.embeddings.patch_embedding.weight.dtype
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
grid_thw: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# SigLIP2 attends to each frame independently, so expand every THW
|
||||
# entry into T separate HW attention sequences.
|
||||
grid_thw_cpu = grid_thw.to(device="cpu")
|
||||
spatial_shapes = torch.repeat_interleave(
|
||||
grid_thw_cpu[:, 1:],
|
||||
grid_thw_cpu[:, 0],
|
||||
dim=0,
|
||||
)
|
||||
lengths_cpu = spatial_shapes.prod(dim=-1).to(torch.int32)
|
||||
lengths = lengths_cpu.to(
|
||||
device=pixel_values.device,
|
||||
non_blocking=True,
|
||||
)
|
||||
|
||||
cu_seqlens = torch.zeros(
|
||||
lengths.numel() + 1,
|
||||
dtype=torch.int32,
|
||||
device=pixel_values.device,
|
||||
)
|
||||
cu_seqlens[1:] = lengths.cumsum(dim=0)
|
||||
|
||||
max_seqlen = lengths_cpu.max().reshape(1)
|
||||
|
||||
return super().forward(
|
||||
pixel_values_packed=pixel_values,
|
||||
spatial_shapes=spatial_shapes,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
)[0]
|
||||
|
||||
|
||||
def patch_merging_by_param(
|
||||
image_embeds: torch.Tensor,
|
||||
grid_thw: torch.Tensor,
|
||||
merge_size: int = 2,
|
||||
) -> torch.Tensor:
|
||||
"""Merge each spatial ``merge_size`` block into the channel dimension."""
|
||||
merged_embeds: list[torch.Tensor] = []
|
||||
current_idx = 0
|
||||
hidden_size = image_embeds.shape[-1]
|
||||
|
||||
for grid in grid_thw:
|
||||
t, h, w = grid.tolist()
|
||||
if h % merge_size != 0 or w % merge_size != 0:
|
||||
raise ValueError(
|
||||
"Grid height and width must be divisible by merge_size: "
|
||||
f"got grid={(t, h, w)} and merge_size={merge_size}."
|
||||
)
|
||||
|
||||
num_patches = t * h * w
|
||||
media_embeds = image_embeds[current_idx : current_idx + num_patches]
|
||||
if media_embeds.shape[0] != num_patches:
|
||||
raise ValueError(
|
||||
"image_embeds contains fewer patches than grid_thw describes."
|
||||
)
|
||||
current_idx += num_patches
|
||||
|
||||
media_embeds = media_embeds.view(t, h, w, hidden_size)
|
||||
media_embeds = media_embeds.view(
|
||||
t,
|
||||
h // merge_size,
|
||||
merge_size,
|
||||
w // merge_size,
|
||||
merge_size,
|
||||
hidden_size,
|
||||
)
|
||||
media_embeds = media_embeds.permute(0, 1, 3, 2, 4, 5).contiguous()
|
||||
merged_embeds.append(
|
||||
media_embeds.view(-1, merge_size * merge_size * hidden_size)
|
||||
)
|
||||
|
||||
if current_idx != image_embeds.shape[0]:
|
||||
raise ValueError(
|
||||
"image_embeds contains more patches than grid_thw describes: "
|
||||
f"got {image_embeds.shape[0]}, expected {current_idx}."
|
||||
)
|
||||
if not merged_embeds:
|
||||
return image_embeds.new_empty((0, merge_size * merge_size * hidden_size))
|
||||
return torch.cat(merged_embeds, dim=0)
|
||||
|
||||
|
||||
class Cosmos3EdgePatchMerger(nn.Module):
|
||||
"""
|
||||
Projector: LayerNorm -> Linear -> GELU -> Linear
|
||||
|
||||
Reads config from projector_config (not vision_config).
|
||||
input_hidden_size * spatial_merge_size² -> merger_intermediate_size
|
||||
-> out_hidden_size
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_hidden_size: int,
|
||||
out_hidden_size: int,
|
||||
merger_intermediate_size: int,
|
||||
spatial_merge_size: int = 2,
|
||||
use_postshuffle_norm: bool = False,
|
||||
quant_config=None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
use_data_parallel = is_vit_use_data_parallel()
|
||||
self.spatial_merge_size = spatial_merge_size
|
||||
self.hidden_size = input_hidden_size * (spatial_merge_size**2)
|
||||
self.input_hidden_size = input_hidden_size
|
||||
self.out_hidden_size = out_hidden_size
|
||||
self.use_postshuffle_norm = use_postshuffle_norm
|
||||
|
||||
norm_dim = self.hidden_size if use_postshuffle_norm else input_hidden_size
|
||||
self.norm = nn.LayerNorm(norm_dim, eps=1e-6)
|
||||
|
||||
self.linear_fc1 = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
merger_intermediate_size,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_fc1",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
self.act_fn = nn.GELU()
|
||||
self.linear_fc2 = RowParallelLinear(
|
||||
merger_intermediate_size,
|
||||
out_hidden_size,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_fc2",
|
||||
disable_tp=use_data_parallel,
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.use_postshuffle_norm:
|
||||
x = self.norm(x.view(-1, self.hidden_size))
|
||||
else:
|
||||
x = self.norm(x).view(-1, self.hidden_size)
|
||||
|
||||
x, _ = self.linear_fc1(x)
|
||||
x = self.act_fn(x)
|
||||
x, _ = self.linear_fc2(x)
|
||||
return x
|
||||
|
||||
|
||||
class Cosmos3EdgeVisionModel(nn.Module):
|
||||
"""Complete Cosmos vision tower returning language-model embeddings."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config,
|
||||
projector_config,
|
||||
quant_config=None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.spatial_merge_size = projector_config.spatial_merge_size
|
||||
self.out_hidden_size = projector_config.out_hidden_size
|
||||
|
||||
self.encoder = Cosmos3EdgeVisionEncoder(
|
||||
vision_config,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "encoder"),
|
||||
)
|
||||
self.projector = Cosmos3EdgePatchMerger(
|
||||
input_hidden_size=projector_config.input_hidden_size,
|
||||
out_hidden_size=projector_config.out_hidden_size,
|
||||
merger_intermediate_size=projector_config.merger_intermediate_size,
|
||||
spatial_merge_size=self.spatial_merge_size,
|
||||
use_postshuffle_norm=getattr(
|
||||
projector_config, "use_postshuffle_norm", False
|
||||
),
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "projector"),
|
||||
)
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return self.encoder.dtype
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
grid_thw: torch.Tensor | list[list[int]],
|
||||
) -> torch.Tensor:
|
||||
grid_thw = torch.as_tensor(grid_thw, dtype=torch.int64, device="cpu")
|
||||
image_embeds = self.encoder(pixel_values.type(self.dtype), grid_thw=grid_thw)
|
||||
image_embeds = patch_merging_by_param(
|
||||
image_embeds,
|
||||
grid_thw,
|
||||
merge_size=self.spatial_merge_size,
|
||||
)
|
||||
image_embeds = image_embeds.view(
|
||||
-1,
|
||||
self.spatial_merge_size**2,
|
||||
self.projector.input_hidden_size,
|
||||
)
|
||||
return self.projector(image_embeds)
|
||||
|
||||
|
||||
class Cosmos3EdgeProcessingInfo(Qwen3VLProcessingInfo):
|
||||
def get_hf_config(self):
|
||||
return self.ctx.get_hf_config()
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
|
||||
return self.ctx.get_hf_processor(**kwargs)
|
||||
|
||||
|
||||
class Cosmos3EdgeMultiModalProcessor(Qwen3VLMultiModalProcessor):
|
||||
pass
|
||||
|
||||
|
||||
class Cosmos3EdgeDummyInputsBuilder(Qwen3VLDummyInputsBuilder):
|
||||
pass
|
||||
|
||||
|
||||
class Cosmos3EdgeAttention(NemotronHAttention):
|
||||
"""Nemotron-H attention with interleaved multimodal RoPE."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: NemotronHConfig,
|
||||
layer_idx: int,
|
||||
model_config=None,
|
||||
cache_config=None,
|
||||
quant_config=None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(
|
||||
config,
|
||||
layer_idx,
|
||||
model_config,
|
||||
cache_config,
|
||||
quant_config,
|
||||
prefix,
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=config.max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class Cosmos3EdgeAttentionDecoderLayer(nn.Module):
|
||||
"""Pre-norm attention layer for the Cosmos3 Edge dense text model."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: NemotronHConfig,
|
||||
layer_idx: int,
|
||||
model_config=None,
|
||||
cache_config=None,
|
||||
quant_config=None,
|
||||
parallel_config=None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
get_layer_config = getattr(config, "get_nemotron_h_config_for_layer", None)
|
||||
layer_config = get_layer_config(layer_idx) if get_layer_config else config
|
||||
self.mixer = Cosmos3EdgeAttention(
|
||||
layer_config,
|
||||
layer_idx,
|
||||
model_config,
|
||||
cache_config,
|
||||
quant_config,
|
||||
prefix=f"{prefix}.mixer",
|
||||
)
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.norm(hidden_states, residual)
|
||||
hidden_states = self.mixer(positions=positions, hidden_states=hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class Cosmos3EdgeTextModel(NemotronHModel):
|
||||
"""Nemotron-H backbone with Cosmos-owned checkpoint name mapping."""
|
||||
|
||||
# The enclosing Cosmos model maps and packs checkpoint weights. Disabling
|
||||
# Nemotron-H's checkpoint-specific hook prevents those names from being
|
||||
# remapped a second time when AutoWeightsLoader descends into this module.
|
||||
load_weights = None # type: ignore[assignment]
|
||||
|
||||
|
||||
class Cosmos3EdgeForCausalLM(nn.Module):
|
||||
"""Minimal CausalLM wrapper for the Cosmos3 Edge dense text model."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
text_config = getattr(hf_config, "text_config", hf_config)
|
||||
vllm_config = vllm_config.with_hf_config(text_config)
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.config = text_config
|
||||
pattern = text_config.hybrid_override_pattern
|
||||
unsupported = set(pattern) - {"*", "-"}
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"Cosmos3 Edge only supports attention (`*`) and MLP (`-`) "
|
||||
f"layers, but found: {sorted(unsupported)}"
|
||||
)
|
||||
if len(pattern) != text_config.num_hidden_layers:
|
||||
raise ValueError(
|
||||
"hybrid_override_pattern must contain one entry per layer: "
|
||||
f"expected {text_config.num_hidden_layers}, "
|
||||
f"got {len(pattern)}"
|
||||
)
|
||||
|
||||
self.model = Cosmos3EdgeTextModel(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
decoder_layer_types={
|
||||
"*": Cosmos3EdgeAttentionDecoderLayer,
|
||||
"-": NemotronHMLPDecoderLayer,
|
||||
},
|
||||
)
|
||||
self.lm_head = ParallelLMHead(
|
||||
text_config.vocab_size,
|
||||
text_config.hidden_size,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(text_config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
return self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
|
||||
def _cosmos3_edge_diffusers_prefix_map() -> dict[str, str]:
|
||||
"""Map Diffusers blocks to interleaved Nemotron-H layers.
|
||||
|
||||
The checkpoint stores attention and MLP in one ``layers.N`` block, while
|
||||
the Nemotron-H model represents them as separate decoder layers. Therefore,
|
||||
checkpoint block ``N`` maps its attention to layer ``2N`` and its MLP to
|
||||
layer ``2N + 1``.
|
||||
"""
|
||||
mappings = {
|
||||
"embed_tokens.": "language_model.model.embed_tokens.",
|
||||
"norm.": "language_model.model.norm_f.",
|
||||
}
|
||||
for physical_idx in range(28):
|
||||
attention_idx = 2 * physical_idx
|
||||
mlp_idx = attention_idx + 1
|
||||
source_prefix = f"layers.{physical_idx}"
|
||||
attention_prefix = f"language_model.model.layers.{attention_idx}"
|
||||
mlp_prefix = f"language_model.model.layers.{mlp_idx}"
|
||||
mappings.update(
|
||||
{
|
||||
f"{source_prefix}.input_layernorm.": f"{attention_prefix}.norm.",
|
||||
f"{source_prefix}.self_attn.qkv_proj.": (
|
||||
f"{attention_prefix}.mixer.qkv_proj."
|
||||
),
|
||||
f"{source_prefix}.self_attn.o_proj.": (
|
||||
f"{attention_prefix}.mixer.o_proj."
|
||||
),
|
||||
f"{source_prefix}.post_attention_layernorm.": (f"{mlp_prefix}.norm."),
|
||||
f"{source_prefix}.mlp.": f"{mlp_prefix}.mixer.",
|
||||
}
|
||||
)
|
||||
return mappings
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
Cosmos3EdgeMultiModalProcessor,
|
||||
info=Cosmos3EdgeProcessingInfo,
|
||||
dummy_inputs=Cosmos3EdgeDummyInputsBuilder,
|
||||
)
|
||||
class Cosmos3EdgeForConditionalGeneration(
|
||||
nn.Module,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
SupportsMRoPE,
|
||||
):
|
||||
"""
|
||||
Cosmos3 Edge model with a SigLIP2 vision encoder.
|
||||
|
||||
Architecture:
|
||||
- self.visual: SigLIP2 encoder + patch merger + projector
|
||||
- self.language_model: Cosmos3EdgeForCausalLM (pure attention + RoPE)
|
||||
"""
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
|
||||
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
|
||||
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
|
||||
},
|
||||
orig_to_new_prefix={
|
||||
**_cosmos3_edge_diffusers_prefix_map(),
|
||||
"proj_in.": None,
|
||||
"proj_out.": None,
|
||||
"time_embedder.": None,
|
||||
"action_proj_in.": None,
|
||||
"action_proj_out.": None,
|
||||
"audio_modality_embed": None,
|
||||
"action_modality_embed": None,
|
||||
"model.visual.": "visual.encoder.",
|
||||
"model.projector.": "visual.projector.",
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
"model.language_model.": "language_model.model.",
|
||||
},
|
||||
orig_to_new_substr={
|
||||
"_moe_gen": None,
|
||||
".add_q_proj.": None,
|
||||
".add_k_proj.": None,
|
||||
".add_v_proj.": None,
|
||||
".to_add_out.": None,
|
||||
".norm_added_q.": None,
|
||||
".norm_added_k.": None,
|
||||
".self_attn.to_q.": ".self_attn.q_proj.",
|
||||
".self_attn.to_k.": ".self_attn.k_proj.",
|
||||
".self_attn.to_v.": ".self_attn.v_proj.",
|
||||
".self_attn.to_out.": ".self_attn.o_proj.",
|
||||
"language_model.embeddings": "language_model.embed_tokens",
|
||||
},
|
||||
)
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
}
|
||||
|
||||
# Cosmos3 Edge unified Diffusers checkpoints store reasoner weights across
|
||||
# transformer/ and vision_encoder/. Match both while excluding VAE weights.
|
||||
allow_patterns_overrides = ["[tv]*er/*.safetensors"]
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("image"):
|
||||
return "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
if modality.startswith("video"):
|
||||
return "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
raise ValueError(f"Unsupported modality: {modality}")
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
|
||||
self.config = config
|
||||
self.multimodal_config = multimodal_config
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
|
||||
with self._mark_tower_model(vllm_config, {"image", "video"}):
|
||||
self.visual = Cosmos3EdgeVisionModel(
|
||||
vision_config=config.vision_config,
|
||||
projector_config=config.projector_config,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = Cosmos3EdgeForCausalLM(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "language_model"),
|
||||
)
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _get_image_features(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
grid_thw: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Run the complete vision tower and split by media item."""
|
||||
image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
|
||||
sizes = (grid_thw.prod(-1) // self.visual.spatial_merge_size**2).tolist()
|
||||
return image_embeds.split(sizes)
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self, **kwargs: object
|
||||
) -> Qwen2_5_VLImageInputs | None:
|
||||
pixel_values = kwargs.pop("pixel_values", None)
|
||||
image_embeds = kwargs.pop("image_embeds", None)
|
||||
image_grid_thw = kwargs.pop("image_grid_thw", None)
|
||||
|
||||
if pixel_values is None and image_embeds is None:
|
||||
return None
|
||||
|
||||
if pixel_values is not None:
|
||||
return Qwen2_5_VLImagePixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values,
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
|
||||
if image_embeds is not None:
|
||||
return Qwen2_5_VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
image_embeds=image_embeds,
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
|
||||
def _parse_and_validate_video_input(
|
||||
self, **kwargs: object
|
||||
) -> Qwen2_5_VLVideoInputs | None:
|
||||
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
|
||||
video_embeds = kwargs.pop("video_embeds", None)
|
||||
video_grid_thw = kwargs.pop("video_grid_thw", None)
|
||||
second_per_grid_ts = kwargs.pop("second_per_grid_ts", None)
|
||||
|
||||
if pixel_values_videos is None and video_embeds is None:
|
||||
return None
|
||||
|
||||
if pixel_values_videos is not None:
|
||||
return Qwen2_5_VLVideoPixelInputs(
|
||||
type="pixel_values_videos",
|
||||
pixel_values_videos=pixel_values_videos,
|
||||
video_grid_thw=video_grid_thw,
|
||||
second_per_grid_ts=second_per_grid_ts,
|
||||
)
|
||||
|
||||
if video_embeds is not None:
|
||||
return Qwen2_5_VLVideoEmbeddingInputs(
|
||||
type="video_embeds",
|
||||
video_embeds=video_embeds,
|
||||
video_grid_thw=video_grid_thw,
|
||||
)
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: Qwen2_5_VLImageInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
grid_thw = image_input["image_grid_thw"]
|
||||
assert grid_thw.ndim == 2
|
||||
|
||||
if image_input["type"] == "image_embeds":
|
||||
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
|
||||
merge_size = self.visual.spatial_merge_size
|
||||
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
|
||||
return image_embeds.split(sizes)
|
||||
|
||||
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
|
||||
if self.use_data_parallel:
|
||||
return run_dp_sharded_mrope_vision_model(
|
||||
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
|
||||
)
|
||||
return self._get_image_features(pixel_values, grid_thw)
|
||||
|
||||
def _process_video_input(
|
||||
self, video_input: Qwen2_5_VLVideoInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
grid_thw = video_input["video_grid_thw"]
|
||||
assert grid_thw.ndim == 2
|
||||
|
||||
if video_input["type"] == "video_embeds":
|
||||
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
|
||||
merge_size = self.visual.spatial_merge_size
|
||||
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
|
||||
return video_embeds.split(sizes)
|
||||
|
||||
pixel_values_videos = video_input["pixel_values_videos"].type(self.visual.dtype)
|
||||
if self.use_data_parallel:
|
||||
return run_dp_sharded_mrope_vision_model(
|
||||
self.visual, pixel_values_videos, grid_thw.tolist(), rope_type="rope_3d"
|
||||
)
|
||||
return self._get_image_features(pixel_values_videos, grid_thw)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
modalities = {}
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
if image_input is not None:
|
||||
modalities["image"] = image_input
|
||||
video_input = self._parse_and_validate_video_input(**kwargs)
|
||||
if video_input is not None:
|
||||
modalities["video"] = video_input
|
||||
return modalities
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
|
||||
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
|
||||
if not mm_input_by_modality:
|
||||
return None
|
||||
|
||||
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
|
||||
|
||||
for modality in mm_input_by_modality:
|
||||
multimodal_input = mm_input_by_modality[modality]
|
||||
if modality == "image":
|
||||
image_embeddings = self._process_image_input(multimodal_input)
|
||||
multimodal_embeddings += tuple(image_embeddings)
|
||||
if modality == "video":
|
||||
video_embeddings = self._process_video_input(multimodal_input)
|
||||
multimodal_embeddings += tuple(video_embeddings)
|
||||
|
||||
return multimodal_embeddings
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
|
||||
return self.language_model.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="visual.projector",
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_mrope_input_positions(
|
||||
self,
|
||||
input_tokens: list[int],
|
||||
mm_features: list[MultiModalFeatureSpec],
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
return Qwen3VLForConditionalGeneration._get_mrope_input_positions(
|
||||
input_tokens=input_tokens,
|
||||
mm_features=mm_features,
|
||||
config=self.config,
|
||||
)
|
||||
@@ -19,7 +19,7 @@
|
||||
"""Inference-only NemotronH model."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
@@ -540,9 +540,22 @@ ALL_DECODER_LAYER_TYPES = {
|
||||
}
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
}
|
||||
)
|
||||
class NemotronHModel(nn.Module, EagleModelMixin):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
decoder_layer_types: Mapping[str, type[nn.Module]] | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
config: NemotronHConfig = vllm_config.model_config.hf_config
|
||||
@@ -562,11 +575,16 @@ class NemotronHModel(nn.Module, EagleModelMixin):
|
||||
|
||||
self.has_moe = "E" in config.hybrid_override_pattern
|
||||
|
||||
layer_types = decoder_layer_types or ALL_DECODER_LAYER_TYPES
|
||||
|
||||
def get_layer(prefix: str):
|
||||
layer_idx = int(prefix.rsplit(".", 1)[1])
|
||||
layer_class = ALL_DECODER_LAYER_TYPES[
|
||||
config.hybrid_override_pattern[layer_idx]
|
||||
]
|
||||
layer_type = config.hybrid_override_pattern[layer_idx]
|
||||
if layer_type not in layer_types:
|
||||
raise ValueError(
|
||||
f"Unsupported layer type {layer_type!r} at layer {layer_idx}"
|
||||
)
|
||||
layer_class = layer_types[layer_type]
|
||||
return layer_class(
|
||||
config=config,
|
||||
layer_idx=layer_idx,
|
||||
|
||||
@@ -349,6 +349,10 @@ _MULTIMODAL_MODELS = {
|
||||
"Cohere2VisionForConditionalGeneration",
|
||||
),
|
||||
"Cosmos3ForConditionalGeneration": ("cosmos3", "Cosmos3ForConditionalGeneration"),
|
||||
"Cosmos3EdgeForConditionalGeneration": (
|
||||
"cosmos3_edge",
|
||||
"Cosmos3EdgeForConditionalGeneration",
|
||||
),
|
||||
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
|
||||
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
|
||||
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
|
||||
|
||||
@@ -81,6 +81,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
ops_colqwen3="OpsColQwen3Config",
|
||||
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
|
||||
cosmos3_omni="Cosmos3Config",
|
||||
cosmos3_edge="Cosmos3EdgeConfig",
|
||||
diffusion_gemma="DiffusionGemmaConfig",
|
||||
deepseek_vl_v2="DeepseekVLV2Config",
|
||||
deepseek_v32="DeepseekV3Config",
|
||||
|
||||
@@ -27,6 +27,10 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
|
||||
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
|
||||
"Cosmos3Config": "vllm.transformers_utils.configs.cosmos3",
|
||||
"Cosmos3EdgeConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"Cosmos3EdgeProjectorConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"Cosmos3EdgeTextConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"Cosmos3EdgeVisionConfig": "vllm.transformers_utils.configs.cosmos3_edge",
|
||||
"DiffusionGemmaConfig": "vllm.transformers_utils.configs.diffusion_gemma",
|
||||
"DiffusionGemmaTextConfig": "vllm.transformers_utils.configs.diffusion_gemma",
|
||||
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
|
||||
@@ -107,6 +111,10 @@ __all__ = [
|
||||
"OpsColQwen3Config",
|
||||
"Qwen3VLNemotronEmbedConfig",
|
||||
"Cosmos3Config",
|
||||
"Cosmos3EdgeConfig",
|
||||
"Cosmos3EdgeProjectorConfig",
|
||||
"Cosmos3EdgeTextConfig",
|
||||
"Cosmos3EdgeVisionConfig",
|
||||
"DiffusionGemmaConfig",
|
||||
"DiffusionGemmaTextConfig",
|
||||
"DeepseekVLV2Config",
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.models.siglip2.configuration_siglip2 import Siglip2VisionConfig
|
||||
|
||||
from .nemotron_h import NemotronHConfig
|
||||
|
||||
|
||||
def _normalize_mrope_parameters(config: NemotronHConfig) -> None:
|
||||
"""Normalize the checkpoint's mRoPE configuration for vLLM."""
|
||||
rope_parameters = dict(getattr(config, "rope_parameters", None) or {})
|
||||
mrope_section = rope_parameters.get("mrope_section")
|
||||
if mrope_section is None:
|
||||
return
|
||||
|
||||
rope_parameters["mrope_section"] = list(mrope_section)
|
||||
rope_parameters.setdefault("mrope_interleaved", True)
|
||||
config.rope_parameters = rope_parameters
|
||||
|
||||
|
||||
class Cosmos3EdgeTextConfig(NemotronHConfig):
|
||||
"""Dense Cosmos3 Edge text config backed by Nemotron-H layers."""
|
||||
|
||||
model_type = "cosmos3_edge_text"
|
||||
has_no_defaults_at_init = True
|
||||
ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_hidden_layers: int = 28,
|
||||
hidden_act: str = "relu2",
|
||||
rms_norm_eps: float = 1e-5,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_hidden_layers=2 * num_hidden_layers,
|
||||
hybrid_override_pattern="*-" * num_hidden_layers,
|
||||
mlp_hidden_act=hidden_act,
|
||||
layer_norm_epsilon=rms_norm_eps,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
config_dict = super().to_dict()
|
||||
config_dict["num_hidden_layers"] = self.num_hidden_layers // 2
|
||||
config_dict["hidden_act"] = config_dict.pop("mlp_hidden_act")
|
||||
config_dict["rms_norm_eps"] = config_dict.pop("layer_norm_epsilon")
|
||||
config_dict.pop("hybrid_override_pattern", None)
|
||||
return config_dict
|
||||
|
||||
|
||||
class Cosmos3EdgeProjectorConfig(PretrainedConfig):
|
||||
model_type = "cosmos3_edge_projector"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_hidden_size: int = 1024,
|
||||
use_postshuffle_norm: bool = False,
|
||||
spatial_merge_size: int = 2,
|
||||
merger_intermediate_size: int = 11520,
|
||||
out_hidden_size: int = 1024,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.input_hidden_size = input_hidden_size
|
||||
self.use_postshuffle_norm = use_postshuffle_norm
|
||||
self.spatial_merge_size = spatial_merge_size
|
||||
self.merger_intermediate_size = merger_intermediate_size
|
||||
self.out_hidden_size = out_hidden_size
|
||||
|
||||
|
||||
class Cosmos3EdgeVisionConfig(Siglip2VisionConfig):
|
||||
model_type = "cosmos3_edge_vision"
|
||||
|
||||
|
||||
class Cosmos3EdgeConfig(PretrainedConfig):
|
||||
model_type = "cosmos3_edge"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
has_no_defaults_at_init = True
|
||||
sub_configs = {
|
||||
"vision_config": Cosmos3EdgeVisionConfig,
|
||||
"projector_config": Cosmos3EdgeProjectorConfig,
|
||||
"text_config": Cosmos3EdgeTextConfig,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: Cosmos3EdgeTextConfig | dict | None = None,
|
||||
vision_config: Cosmos3EdgeVisionConfig | dict | None = None,
|
||||
projector_config: Cosmos3EdgeProjectorConfig | dict | None = None,
|
||||
image_token_id: int = 19,
|
||||
video_token_id: int = 18,
|
||||
vision_start_token_id: int = 20,
|
||||
vision_end_token_id: int = 21,
|
||||
tie_word_embeddings: bool = False,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
if text_config is None:
|
||||
text_config = {}
|
||||
if vision_config is None:
|
||||
vision_config = {}
|
||||
if projector_config is None:
|
||||
projector_config = {}
|
||||
|
||||
self.text_config = (
|
||||
Cosmos3EdgeTextConfig(**text_config)
|
||||
if isinstance(text_config, dict)
|
||||
else text_config
|
||||
)
|
||||
_normalize_mrope_parameters(self.text_config)
|
||||
|
||||
self.vision_config = (
|
||||
Cosmos3EdgeVisionConfig(**vision_config)
|
||||
if isinstance(vision_config, dict)
|
||||
else vision_config
|
||||
)
|
||||
self.projector_config = (
|
||||
Cosmos3EdgeProjectorConfig(**projector_config)
|
||||
if isinstance(projector_config, dict)
|
||||
else projector_config
|
||||
)
|
||||
|
||||
# Qwen3-VL processing and M-RoPE read these attributes from the
|
||||
# vision config, while this checkpoint stores them in projector config.
|
||||
self.vision_config.spatial_merge_size = self.projector_config.spatial_merge_size
|
||||
self.vision_config.temporal_patch_size = 1
|
||||
self.vision_config.out_hidden_size = self.projector_config.out_hidden_size
|
||||
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.vision_start_token_id = vision_start_token_id
|
||||
self.vision_end_token_id = vision_end_token_id
|
||||
|
||||
super().__init__(
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Cosmos3EdgeConfig",
|
||||
"Cosmos3EdgeProjectorConfig",
|
||||
"Cosmos3EdgeTextConfig",
|
||||
"Cosmos3EdgeVisionConfig",
|
||||
]
|
||||
@@ -14,6 +14,7 @@ __all__ = [
|
||||
"BagelProcessor",
|
||||
"CheersProcessor",
|
||||
"CohereASRProcessor",
|
||||
"Cosmos3EdgeProcessor",
|
||||
"DeepseekVLV2Processor",
|
||||
"FireRedASR2Processor",
|
||||
"FireRedLIDProcessor",
|
||||
@@ -49,6 +50,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"BagelProcessor": "vllm.transformers_utils.processors.bagel",
|
||||
"CheersProcessor": "vllm.transformers_utils.processors.cheers",
|
||||
"CohereASRProcessor": "vllm.transformers_utils.processors.cohere_asr",
|
||||
"Cosmos3EdgeProcessor": "vllm.transformers_utils.processors.cosmos3_edge",
|
||||
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
|
||||
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
|
||||
"FireRedLIDProcessor": "vllm.transformers_utils.processors.fireredlid",
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
from torchvision.transforms import InterpolationMode
|
||||
from transformers import AutoTokenizer
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.image_utils import ChannelDimension, PILImageResampling, SizeDict
|
||||
from transformers.models.qwen3_vl.processing_qwen3_vl import Qwen3VLProcessor
|
||||
from transformers.models.qwen3_vl.video_processing_qwen3_vl import (
|
||||
Qwen3VLVideoProcessor,
|
||||
Qwen3VLVideoProcessorInitKwargs,
|
||||
get_image_size,
|
||||
smart_resize,
|
||||
)
|
||||
from transformers.models.siglip2.image_processing_siglip2 import (
|
||||
Siglip2ImageProcessor,
|
||||
convert_image_to_patches,
|
||||
)
|
||||
from transformers.processing_utils import Unpack
|
||||
from transformers.utils import TensorType
|
||||
from transformers.video_utils import group_videos_by_shape, reorder_videos
|
||||
|
||||
|
||||
def round_by_factor(number: int, factor: int) -> int:
|
||||
"""Returns the closest integer to 'number' that is divisible by 'factor'."""
|
||||
return round(number / factor) * factor
|
||||
|
||||
|
||||
def ceil_by_factor(number: int, factor: int) -> int:
|
||||
"""Return the smallest multiple of ``factor`` greater than ``number``."""
|
||||
return math.ceil(number / factor) * factor
|
||||
|
||||
|
||||
def floor_by_factor(number: int, factor: int) -> int:
|
||||
"""Return the largest multiple of ``factor`` less than ``number``."""
|
||||
return math.floor(number / factor) * factor
|
||||
|
||||
|
||||
class Cosmos3EdgeImagesKwargs( # type: ignore[call-arg]
|
||||
Siglip2ImageProcessor.valid_kwargs,
|
||||
total=False,
|
||||
):
|
||||
# global setting for all images, can be overridden by per-image kwargs
|
||||
max_pixels: int | None
|
||||
min_pixels: int | None
|
||||
|
||||
per_image_kwargs: list[dict] | None
|
||||
|
||||
|
||||
class Cosmos3EdgeImageProcessor(Siglip2ImageProcessor):
|
||||
resample = PILImageResampling.BICUBIC
|
||||
valid_kwargs = Cosmos3EdgeImagesKwargs
|
||||
|
||||
def __init__(self, **kwargs: Unpack[Cosmos3EdgeImagesKwargs]):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def _resize_image(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
max_ratio=200,
|
||||
resample: PILImageResampling | InterpolationMode | int | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Rescales the image so that the following conditions are met:
|
||||
|
||||
1. Both dimensions (height and width) are divisible by 'factor'.
|
||||
2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
|
||||
3. The aspect ratio of the image is maintained as closely as possible.
|
||||
"""
|
||||
image_min_pixels = kwargs.get("min_pixels") or self.size.get(
|
||||
"shortest_edge", None
|
||||
)
|
||||
image_max_pixels = kwargs.get("max_pixels") or self.size.get(
|
||||
"longest_edge", None
|
||||
)
|
||||
assert image_min_pixels is not None and image_max_pixels is not None, (
|
||||
"When do_resize is True, min_pixels and max_pixels must be provided."
|
||||
)
|
||||
assert image_max_pixels >= image_min_pixels, (
|
||||
"The max_pixels of image must be greater than or equal to min_pixels."
|
||||
)
|
||||
|
||||
_, height, width = image.shape
|
||||
aspect_ratio = max(height, width) / min(height, width)
|
||||
if aspect_ratio > max_ratio:
|
||||
raise ValueError(
|
||||
f"absolute aspect ratio must be smaller than {max_ratio}, "
|
||||
f"got {aspect_ratio}"
|
||||
)
|
||||
factor = self.merge_size * self.patch_size
|
||||
h_bar = max(factor, round_by_factor(height, factor))
|
||||
w_bar = max(factor, round_by_factor(width, factor))
|
||||
if h_bar * w_bar > image_max_pixels:
|
||||
beta = math.sqrt((height * width) / image_max_pixels)
|
||||
h_bar = floor_by_factor(height / beta, factor)
|
||||
w_bar = floor_by_factor(width / beta, factor)
|
||||
elif h_bar * w_bar < image_min_pixels:
|
||||
beta = math.sqrt(image_min_pixels / (height * width))
|
||||
h_bar = ceil_by_factor(height * beta, factor)
|
||||
w_bar = ceil_by_factor(width * beta, factor)
|
||||
|
||||
image = self.resize(
|
||||
image,
|
||||
size=SizeDict(height=h_bar, width=w_bar),
|
||||
resample=resample,
|
||||
)
|
||||
return image
|
||||
|
||||
def _preprocess(
|
||||
self,
|
||||
images: list["torch.Tensor"],
|
||||
do_resize: bool,
|
||||
patch_size: int,
|
||||
max_num_patches: int,
|
||||
resample: PILImageResampling | InterpolationMode | int | None,
|
||||
do_rescale: bool,
|
||||
rescale_factor: float,
|
||||
do_normalize: bool,
|
||||
image_mean: float | list[float] | None,
|
||||
image_std: float | list[float] | None,
|
||||
return_tensors: str | TensorType | None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
per_image_kwargs = kwargs.pop("per_image_kwargs", None)
|
||||
pixel_values = []
|
||||
spatial_shapes = []
|
||||
for idx, image in enumerate(images):
|
||||
# merge global kwargs with per-image overrides (if provided)
|
||||
image_kwargs = kwargs.copy()
|
||||
if (
|
||||
per_image_kwargs is not None
|
||||
and idx < len(per_image_kwargs)
|
||||
and per_image_kwargs[idx]
|
||||
):
|
||||
image_kwargs.update(per_image_kwargs[idx])
|
||||
if do_resize:
|
||||
image = self._resize_image(
|
||||
image,
|
||||
max_ratio=200,
|
||||
resample=resample,
|
||||
**image_kwargs,
|
||||
)
|
||||
|
||||
image = self.rescale_and_normalize(
|
||||
image, do_rescale, rescale_factor, do_normalize, image_mean, image_std
|
||||
)
|
||||
|
||||
# (channels, height, width) -> (patches, patch_size**2 * channels)
|
||||
patches = convert_image_to_patches(image, patch_size)
|
||||
|
||||
num_patches_height = image.shape[1] // patch_size
|
||||
num_patches_width = image.shape[2] // patch_size
|
||||
|
||||
spatial_shapes.append((num_patches_height, num_patches_width))
|
||||
pixel_values.append(patches)
|
||||
|
||||
pixel_values = torch.cat(pixel_values, dim=0)
|
||||
|
||||
spatial_shapes = torch.tensor(spatial_shapes)
|
||||
t_dim = torch.ones(
|
||||
(spatial_shapes.shape[0], 1),
|
||||
dtype=spatial_shapes.dtype,
|
||||
device=spatial_shapes.device,
|
||||
)
|
||||
image_grid_thw = torch.cat([t_dim, spatial_shapes], dim=1)
|
||||
|
||||
batch_feature = BatchFeature(
|
||||
data={
|
||||
"pixel_values": pixel_values,
|
||||
"image_grid_thw": image_grid_thw,
|
||||
},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
return batch_feature
|
||||
|
||||
|
||||
class Cosmos3EdgeVideoProcessor(Qwen3VLVideoProcessor):
|
||||
def __init__(self, **kwargs: Unpack[Qwen3VLVideoProcessorInitKwargs]):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def _preprocess(
|
||||
self,
|
||||
videos: list[torch.Tensor],
|
||||
do_convert_rgb: bool = True,
|
||||
do_resize: bool = True,
|
||||
size: SizeDict | None = None,
|
||||
resample: PILImageResampling
|
||||
| InterpolationMode
|
||||
| int
|
||||
| None = PILImageResampling.BICUBIC,
|
||||
do_rescale: bool = True,
|
||||
rescale_factor: float = 1 / 255.0,
|
||||
do_normalize: bool = True,
|
||||
image_mean: float | list[float] | None = None,
|
||||
image_std: float | list[float] | None = None,
|
||||
patch_size: int | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
assert patch_size is not None
|
||||
|
||||
merge_size = self.merge_size
|
||||
grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
|
||||
resized_videos_grouped = {}
|
||||
|
||||
for shape, stacked_videos in grouped_videos.items():
|
||||
B, T, C, H, W = stacked_videos.shape
|
||||
num_frames, height, width = T, H, W
|
||||
if do_resize:
|
||||
assert size is not None
|
||||
# 1. Both dimensions (height and width) are divisible by 'factor'.
|
||||
# 2. Total pixels are within the configured min/max range.
|
||||
resized_height, resized_width = smart_resize(
|
||||
num_frames=num_frames,
|
||||
height=height,
|
||||
width=width,
|
||||
temporal_factor=1,
|
||||
factor=patch_size * merge_size,
|
||||
min_pixels=size.shortest_edge,
|
||||
max_pixels=size.longest_edge,
|
||||
)
|
||||
stacked_videos = stacked_videos.view(B * T, C, H, W)
|
||||
stacked_videos = self.resize(
|
||||
stacked_videos,
|
||||
size=SizeDict(height=resized_height, width=resized_width),
|
||||
resample=resample,
|
||||
)
|
||||
stacked_videos = stacked_videos.view(
|
||||
B, T, C, resized_height, resized_width
|
||||
)
|
||||
resized_videos_grouped[shape] = stacked_videos
|
||||
resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
|
||||
|
||||
# Group videos by size for further processing
|
||||
# Resize may return videos with different sizes.
|
||||
grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
|
||||
processed_videos_grouped = {}
|
||||
processed_grids = {}
|
||||
for shape, stacked_videos in grouped_videos.items():
|
||||
resized_height, resized_width = get_image_size(
|
||||
stacked_videos[0], channel_dim=ChannelDimension.FIRST
|
||||
)
|
||||
# Fused rescale and normalize
|
||||
stacked_videos = self.rescale_and_normalize(
|
||||
stacked_videos,
|
||||
do_rescale,
|
||||
rescale_factor,
|
||||
do_normalize,
|
||||
image_mean,
|
||||
image_std,
|
||||
)
|
||||
patches = stacked_videos
|
||||
|
||||
batch_size, grid_t, channel = patches.shape[:3]
|
||||
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
|
||||
|
||||
# [1, 16, 3, 320, 608] -> [1, 16, 3, 20, 16, 38, 16]
|
||||
patches = patches.view(
|
||||
batch_size, # 0
|
||||
grid_t, # 1
|
||||
channel, # 2
|
||||
grid_h, # 3
|
||||
patch_size, # 4
|
||||
grid_w, # 5
|
||||
patch_size, # 6
|
||||
)
|
||||
# [1, 16, 3, 20, 16, 38, 16] -> [1, 16, 20, 38, 16, 16, 3]
|
||||
# Result: [batch, grid_t, grid_h, grid_w, patch_h, patch_w, channel]
|
||||
patches = patches.permute(0, 1, 3, 5, 4, 6, 2)
|
||||
|
||||
# [1, 16, 20, 38, 16, 16, 3] -> [1, 5120, 768]
|
||||
flatten_patches = patches.reshape(
|
||||
batch_size,
|
||||
grid_t * grid_h * grid_w,
|
||||
patch_size * patch_size * channel,
|
||||
)
|
||||
|
||||
processed_videos_grouped[shape] = flatten_patches
|
||||
processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
|
||||
|
||||
processed_videos = reorder_videos(
|
||||
processed_videos_grouped, grouped_videos_index
|
||||
)
|
||||
processed_grids = reorder_videos(processed_grids, grouped_videos_index)
|
||||
pixel_values_videos = torch.cat(processed_videos, dim=0)
|
||||
video_grid_thw = torch.tensor(processed_grids)
|
||||
data = {
|
||||
"pixel_values_videos": pixel_values_videos,
|
||||
"video_grid_thw": video_grid_thw,
|
||||
}
|
||||
|
||||
return BatchFeature(data=data, tensor_type=return_tensors)
|
||||
|
||||
|
||||
class Cosmos3EdgeProcessor(Qwen3VLProcessor):
|
||||
"""Construct a processor for Cosmos3 Edge multimodal inputs.
|
||||
|
||||
This processor combines :class:`Cosmos3EdgeImageProcessor`,
|
||||
:class:`Cosmos3EdgeVideoProcessor`, and a tokenizer into a single
|
||||
interface. It inherits the text, image, and video input handling and the
|
||||
decoding methods from :class:`Qwen3VLProcessor`, while using the Cosmos3
|
||||
Edge-specific image and video preprocessing implementations.
|
||||
|
||||
Args:
|
||||
image_processor: A :class:`Cosmos3EdgeImageProcessor` used to resize,
|
||||
normalize, and patch image inputs.
|
||||
tokenizer: The tokenizer used to encode and decode text.
|
||||
video_processor: A :class:`Cosmos3EdgeVideoProcessor` used to resize,
|
||||
normalize, and patch video inputs.
|
||||
chat_template: An optional Jinja template used to render chat messages
|
||||
into a tokenizable string.
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer", "video_processor"]
|
||||
|
||||
# Transformers' auto loaders cannot resolve these vLLM-native image and
|
||||
# video processors without remote code or global auto-class registration.
|
||||
@classmethod
|
||||
def _get_arguments_from_pretrained(
|
||||
cls,
|
||||
pretrained_model_name_or_path,
|
||||
processor_dict=None,
|
||||
**kwargs,
|
||||
):
|
||||
component_kwargs = dict(kwargs)
|
||||
component_kwargs.pop("trust_remote_code", None)
|
||||
|
||||
image_processor = Cosmos3EdgeImageProcessor.from_pretrained(
|
||||
pretrained_model_name_or_path,
|
||||
**component_kwargs,
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
pretrained_model_name_or_path,
|
||||
trust_remote_code=False,
|
||||
**component_kwargs,
|
||||
)
|
||||
video_processor = Cosmos3EdgeVideoProcessor.from_pretrained(
|
||||
pretrained_model_name_or_path,
|
||||
**component_kwargs,
|
||||
)
|
||||
|
||||
return [
|
||||
image_processor,
|
||||
tokenizer,
|
||||
video_processor,
|
||||
]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Cosmos3EdgeProcessor",
|
||||
]
|
||||
Reference in New Issue
Block a user