[Model] Openvla support (#42654)
Signed-off-by: Wang Yiwen <121547057+yiwen101@users.noreply.github.com>
This commit is contained in:
@@ -120,6 +120,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
speculators="SpeculatorsConfig",
|
||||
nemotron="NemotronConfig",
|
||||
olmo_hybrid="OlmoHybridConfig",
|
||||
openvla="OpenVLAConfig",
|
||||
ovis="OvisConfig",
|
||||
ultravox="UltravoxConfig",
|
||||
step3_vl="Step3VLConfig",
|
||||
|
||||
@@ -61,6 +61,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"NemotronConfig": "vllm.transformers_utils.configs.nemotron",
|
||||
"NemotronHConfig": "vllm.transformers_utils.configs.nemotron_h",
|
||||
"OlmoHybridConfig": "vllm.transformers_utils.configs.olmo_hybrid",
|
||||
"OpenVLAConfig": "vllm.transformers_utils.configs.openvla",
|
||||
"OvisConfig": "vllm.transformers_utils.configs.ovis",
|
||||
"PixelShuffleSiglip2VisionConfig": "vllm.transformers_utils.configs.isaac",
|
||||
"RadioConfig": "vllm.transformers_utils.configs.radio",
|
||||
@@ -128,6 +129,7 @@ __all__ = [
|
||||
"NemotronConfig",
|
||||
"NemotronHConfig",
|
||||
"OlmoHybridConfig",
|
||||
"OpenVLAConfig",
|
||||
"OvisConfig",
|
||||
"PixelShuffleSiglip2VisionConfig",
|
||||
"RadioConfig",
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""OpenVLA configuration support.
|
||||
|
||||
OpenVLA checkpoints use a custom ``model_type`` and nest the language model
|
||||
configuration under ``text_config``. This shim lets vLLM load the checkpoint
|
||||
configuration without executing Hugging Face remote code.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from transformers import LlamaConfig, PretrainedConfig
|
||||
|
||||
|
||||
class OpenVLAConfig(PretrainedConfig):
|
||||
"""Configuration class for OpenVLA models."""
|
||||
|
||||
model_type = "openvla"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
timm_model_ids: list[str] | None = None,
|
||||
timm_override_act_layers: list[str | None] | None = None,
|
||||
image_sizes: list[int] | None = None,
|
||||
use_fused_vision_backbone: bool = True,
|
||||
image_token_index: int = 32000,
|
||||
n_action_bins: int = 256,
|
||||
text_config: dict[str, Any] | LlamaConfig | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
kwargs.setdefault("architectures", ["OpenVLAForActionPrediction"])
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.timm_model_ids = timm_model_ids or [
|
||||
"vit_large_patch14_reg4_dinov2.lvd142m",
|
||||
"vit_so400m_patch14_siglip_224",
|
||||
]
|
||||
self.timm_override_act_layers = timm_override_act_layers or [None, None]
|
||||
self.image_sizes = image_sizes or [224, 224]
|
||||
self.use_fused_vision_backbone = use_fused_vision_backbone
|
||||
self.image_token_index = image_token_index
|
||||
self.n_action_bins = n_action_bins
|
||||
|
||||
if text_config is None:
|
||||
text_config = LlamaConfig(architectures=["LlamaForCausalLM"])
|
||||
elif isinstance(text_config, dict):
|
||||
text_config = text_config.copy()
|
||||
text_config.setdefault("architectures", ["LlamaForCausalLM"])
|
||||
text_config = LlamaConfig(**text_config)
|
||||
self.text_config = text_config
|
||||
@@ -35,6 +35,7 @@ __all__ = [
|
||||
"NemotronVLProcessor",
|
||||
"LlamaNemotronVLEmbedProcessor",
|
||||
"NVLMProcessor",
|
||||
"OpenVLAProcessor",
|
||||
"OvisProcessor",
|
||||
"Ovis2_5Processor",
|
||||
"QwenVLProcessor",
|
||||
@@ -67,6 +68,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"NemotronVLProcessor": "vllm.transformers_utils.processors.nemotron_vl",
|
||||
"LlamaNemotronVLEmbedProcessor": "vllm.transformers_utils.processors.nemotron_vl",
|
||||
"NVLMProcessor": "vllm.transformers_utils.processors.nvlm_d",
|
||||
"OpenVLAProcessor": "vllm.transformers_utils.processors.openvla",
|
||||
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
|
||||
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
|
||||
"QwenVLProcessor": "vllm.transformers_utils.processors.qwen_vl",
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
IMAGENET_MEAN = np.array([0.484375, 0.455078125, 0.40625], dtype=np.float32)
|
||||
IMAGENET_STD = np.array([0.228515625, 0.2236328125, 0.224609375], dtype=np.float32)
|
||||
SIGLIP_MEAN = np.array([0.5, 0.5, 0.5], dtype=np.float32)
|
||||
SIGLIP_STD = np.array([0.5, 0.5, 0.5], dtype=np.float32)
|
||||
|
||||
|
||||
def to_rgb_image(image: Any) -> Image.Image:
|
||||
if isinstance(image, Image.Image):
|
||||
return image.convert("RGB")
|
||||
|
||||
if isinstance(image, torch.Tensor):
|
||||
image = image.detach().cpu().numpy()
|
||||
if not isinstance(image, np.ndarray):
|
||||
raise TypeError(
|
||||
"OpenVLA image input must be a PIL image, numpy array, or torch tensor; "
|
||||
f"got {type(image)}"
|
||||
)
|
||||
|
||||
if image.ndim != 3:
|
||||
raise ValueError(
|
||||
f"OpenVLA image input must have 3 dimensions, got shape {image.shape}"
|
||||
)
|
||||
|
||||
if image.shape[0] in (1, 3):
|
||||
image = np.moveaxis(image, 0, -1)
|
||||
|
||||
if image.shape[-1] == 1:
|
||||
image = np.repeat(image, 3, axis=-1)
|
||||
elif image.shape[-1] != 3:
|
||||
raise ValueError(
|
||||
f"OpenVLA image input must have 1 or 3 channels, got shape {image.shape}"
|
||||
)
|
||||
|
||||
if image.dtype != np.uint8:
|
||||
image = image.astype(np.float32)
|
||||
if image.max(initial=0.0) <= 1.0:
|
||||
image = image * 255.0
|
||||
image = np.clip(image, 0, 255).astype(np.uint8)
|
||||
|
||||
return Image.fromarray(image).convert("RGB")
|
||||
|
||||
|
||||
def preprocess_openvla_image(image: Any, image_size: int) -> torch.Tensor:
|
||||
rgb_image = to_rgb_image(image)
|
||||
rgb_image = rgb_image.resize(
|
||||
(image_size, image_size),
|
||||
Image.Resampling.BICUBIC,
|
||||
)
|
||||
|
||||
raw = np.asarray(rgb_image, dtype=np.float32) / 255.0
|
||||
dinov2_pixels = ((raw - IMAGENET_MEAN) / IMAGENET_STD).transpose(2, 0, 1)
|
||||
siglip_pixels = ((raw - SIGLIP_MEAN) / SIGLIP_STD).transpose(2, 0, 1)
|
||||
pixel_values = np.concatenate([dinov2_pixels, siglip_pixels], axis=0)
|
||||
return torch.from_numpy(pixel_values)
|
||||
|
||||
|
||||
class OpenVLAImageProcessor:
|
||||
def __init__(self, *, image_size: int) -> None:
|
||||
self.image_size = image_size
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: Any | None = None,
|
||||
**kwargs: object,
|
||||
) -> dict[str, object]:
|
||||
if images is None:
|
||||
return {}
|
||||
if not isinstance(images, Sequence) or isinstance(images, (str, bytes)):
|
||||
images = [images]
|
||||
if len(images) == 0:
|
||||
return {}
|
||||
|
||||
pixel_values = torch.stack(
|
||||
[
|
||||
preprocess_openvla_image(image, image_size=self.image_size)
|
||||
for image in images
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
return {"pixel_values": pixel_values}
|
||||
|
||||
|
||||
class OpenVLAProcessor(ProcessorMixin):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
image_processor: OpenVLAImageProcessor,
|
||||
tokenizer: Any,
|
||||
) -> None:
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
Reference in New Issue
Block a user