[Mypy] Fix mypy typing for L models (#54177)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -113,7 +113,6 @@ SEPARATE_GROUPS = [
|
|||||||
|
|
||||||
EXCLUDE = [
|
EXCLUDE = [
|
||||||
r"vllm/model_executor/models/[kK]",
|
r"vllm/model_executor/models/[kK]",
|
||||||
r"vllm/model_executor/models/[lL]",
|
|
||||||
r"vllm/model_executor/models/[nN]",
|
r"vllm/model_executor/models/[nN]",
|
||||||
r"vllm/model_executor/models/[oO]",
|
r"vllm/model_executor/models/[oO]",
|
||||||
r"vllm/model_executor/models/[pP]",
|
r"vllm/model_executor/models/[pP]",
|
||||||
|
|||||||
@@ -317,6 +317,7 @@ class LagunaAttention(nn.Module):
|
|||||||
# config.gating may be:
|
# config.gating may be:
|
||||||
# - True / "per-element": one gate per (head, head_dim) channel
|
# - True / "per-element": one gate per (head, head_dim) channel
|
||||||
# - "per-head": one gate per head, broadcast across head_dim
|
# - "per-head": one gate per head, broadcast across head_dim
|
||||||
|
self.g_proj: ColumnParallelLinear | None
|
||||||
if self.gating:
|
if self.gating:
|
||||||
# v5 LagunaConfig uses ``gating=True`` for per-head; older configs
|
# v5 LagunaConfig uses ``gating=True`` for per-head; older configs
|
||||||
# used ``"per-head"``. Accept both. ``"per-element"`` (or legacy
|
# used ``"per-head"``. Accept both. ``"per-element"`` (or legacy
|
||||||
|
|||||||
@@ -69,7 +69,9 @@ class DFlashLagunaModel(DFlashQwen3Model, EagleModelMixin):
|
|||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
) -> None:
|
) -> None:
|
||||||
nn.Module.__init__(self)
|
nn.Module.__init__(self)
|
||||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
speculative_config = vllm_config.speculative_config
|
||||||
|
assert speculative_config is not None
|
||||||
|
self.config = speculative_config.draft_model_config.hf_config
|
||||||
self.vocab_size = self.config.vocab_size
|
self.vocab_size = self.config.vocab_size
|
||||||
self.quant_config = get_draft_quant_config(vllm_config)
|
self.quant_config = get_draft_quant_config(vllm_config)
|
||||||
|
|
||||||
@@ -216,24 +218,16 @@ class DFlashLagunaModel(DFlashQwen3Model, EagleModelMixin):
|
|||||||
return all_k_normed
|
return all_k_normed
|
||||||
|
|
||||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||||
|
if self.quant_config is not None:
|
||||||
|
weights = self.quant_config.get_cache_scale_mapper().apply(weights)
|
||||||
params_dict = dict(self.named_parameters())
|
params_dict = dict(self.named_parameters())
|
||||||
loaded_params: set[str] = set()
|
loaded_params: set[str] = set()
|
||||||
for name, loaded_weight in weights:
|
for name, loaded_weight in weights:
|
||||||
if self.quant_config is not None and (
|
|
||||||
scale_name := self.quant_config.get_cache_scale(name)
|
|
||||||
):
|
|
||||||
param = params_dict[scale_name]
|
|
||||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
|
||||||
loaded_weight = (
|
|
||||||
loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
|
|
||||||
)
|
|
||||||
weight_loader(param, loaded_weight)
|
|
||||||
loaded_params.add(scale_name)
|
|
||||||
continue
|
|
||||||
if "scale" in name:
|
if "scale" in name:
|
||||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||||
if name is None:
|
if remapped_name is None:
|
||||||
continue
|
continue
|
||||||
|
name = remapped_name
|
||||||
param = params_dict[name]
|
param = params_dict[name]
|
||||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||||
weight_loader(param, loaded_weight)
|
weight_loader(param, loaded_weight)
|
||||||
@@ -244,7 +238,9 @@ class DFlashLagunaModel(DFlashQwen3Model, EagleModelMixin):
|
|||||||
class DFlashLagunaForCausalLM(nn.Module, SupportsEagle3):
|
class DFlashLagunaForCausalLM(nn.Module, SupportsEagle3):
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||||
nn.Module.__init__(self)
|
nn.Module.__init__(self)
|
||||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
speculative_config = vllm_config.speculative_config
|
||||||
|
assert speculative_config is not None
|
||||||
|
self.config = speculative_config.draft_model_config.hf_config
|
||||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||||
raise ValueError("Laguna DFlash config requires `draft_vocab_size`.")
|
raise ValueError("Laguna DFlash config requires `draft_vocab_size`.")
|
||||||
self.has_own_embed_tokens = False
|
self.has_own_embed_tokens = False
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
import itertools
|
import itertools
|
||||||
import math
|
import math
|
||||||
|
import typing
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from typing import Annotated, Any, Literal
|
from typing import Annotated, Any, Literal
|
||||||
|
|
||||||
@@ -17,6 +18,7 @@ from transformers.models.lfm2_vl.image_processing_lfm2_vl_fast import (
|
|||||||
find_closest_aspect_ratio,
|
find_closest_aspect_ratio,
|
||||||
round_by_factor,
|
round_by_factor,
|
||||||
)
|
)
|
||||||
|
from typing_extensions import Buffer
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions
|
||||||
@@ -229,6 +231,13 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
|
|||||||
"max_image_tokens", image_processor.max_image_tokens
|
"max_image_tokens", image_processor.max_image_tokens
|
||||||
)
|
)
|
||||||
tile_size = mm_kwargs.get("tile_size", image_processor.tile_size)
|
tile_size = mm_kwargs.get("tile_size", image_processor.tile_size)
|
||||||
|
assert isinstance(downsample_factor, int)
|
||||||
|
assert isinstance(encoder_patch_size, int)
|
||||||
|
assert isinstance(max_pixels_tolerance, int | float)
|
||||||
|
assert isinstance(min_tiles, int)
|
||||||
|
assert isinstance(max_tiles, int)
|
||||||
|
assert isinstance(max_image_tokens, int)
|
||||||
|
assert isinstance(tile_size, int)
|
||||||
|
|
||||||
do_image_splitting = not min_tiles == max_tiles == 1
|
do_image_splitting = not min_tiles == max_tiles == 1
|
||||||
is_image_large = self._is_image_too_large(
|
is_image_large = self._is_image_too_large(
|
||||||
@@ -337,6 +346,9 @@ class Lfm2VLProcessingInfo(BaseProcessingInfo):
|
|||||||
"encoder_patch_size", image_processor.encoder_patch_size
|
"encoder_patch_size", image_processor.encoder_patch_size
|
||||||
)
|
)
|
||||||
tile_size = mm_kwargs.get("tile_size", image_processor.tile_size)
|
tile_size = mm_kwargs.get("tile_size", image_processor.tile_size)
|
||||||
|
assert isinstance(downsample_factor, int)
|
||||||
|
assert isinstance(encoder_patch_size, int)
|
||||||
|
assert isinstance(tile_size, int)
|
||||||
|
|
||||||
thumbnail_height_patches = int(spatial_shapes[-1][0].item())
|
thumbnail_height_patches = int(spatial_shapes[-1][0].item())
|
||||||
thumbnail_width_patches = int(spatial_shapes[-1][1].item())
|
thumbnail_width_patches = int(spatial_shapes[-1][1].item())
|
||||||
@@ -672,6 +684,7 @@ class Lfm2VLForConditionalGeneration(
|
|||||||
super().__init__()
|
super().__init__()
|
||||||
config: Lfm2VlConfig = vllm_config.model_config.hf_config
|
config: Lfm2VlConfig = vllm_config.model_config.hf_config
|
||||||
multimodal_config = vllm_config.model_config.multimodal_config
|
multimodal_config = vllm_config.model_config.multimodal_config
|
||||||
|
assert multimodal_config is not None
|
||||||
vision_config = config.vision_config
|
vision_config = config.vision_config
|
||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
|
|
||||||
@@ -915,6 +928,14 @@ class Lfm2VLForConditionalGeneration(
|
|||||||
"min_image_tokens",
|
"min_image_tokens",
|
||||||
getattr(self.config, "min_image_tokens", None) or 64,
|
getattr(self.config, "min_image_tokens", None) or 64,
|
||||||
)
|
)
|
||||||
|
if not isinstance(
|
||||||
|
value,
|
||||||
|
(str, Buffer, typing.SupportsInt, typing.SupportsIndex),
|
||||||
|
):
|
||||||
|
raise TypeError(
|
||||||
|
"int() argument must be a string, a bytes-like object "
|
||||||
|
f"or a real number, not '{type(value).__name__}'"
|
||||||
|
)
|
||||||
return max(1, int(value))
|
return max(1, int(value))
|
||||||
|
|
||||||
def _get_lfm2vl_item_tile_slices(
|
def _get_lfm2vl_item_tile_slices(
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
|
|||||||
return PixtralHFEncoderInfo(self.get_hf_config())
|
return PixtralHFEncoderInfo(self.get_hf_config())
|
||||||
|
|
||||||
|
|
||||||
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]):
|
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,7 @@
|
|||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from itertools import islice
|
from itertools import islice
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
@@ -544,13 +545,25 @@ class LlamaForCausalLM(
|
|||||||
return loader.load_weights(weights)
|
return loader.load_weights(weights)
|
||||||
|
|
||||||
|
|
||||||
class LlamaBidirectionalForSequenceClassification(as_seq_cls_model(LlamaForCausalLM)):
|
if TYPE_CHECKING:
|
||||||
|
_LlamaBidirectionalForSequenceClassificationBase = LlamaForCausalLM
|
||||||
|
_LlamaBidirectionalModelBase = LlamaForCausalLM
|
||||||
|
else:
|
||||||
|
_LlamaBidirectionalForSequenceClassificationBase = as_seq_cls_model(
|
||||||
|
LlamaForCausalLM
|
||||||
|
)
|
||||||
|
_LlamaBidirectionalModelBase = as_embedding_model(LlamaForCausalLM)
|
||||||
|
|
||||||
|
|
||||||
|
class LlamaBidirectionalForSequenceClassification(
|
||||||
|
_LlamaBidirectionalForSequenceClassificationBase
|
||||||
|
):
|
||||||
# This class sets the correct attention type and pooling type
|
# This class sets the correct attention type and pooling type
|
||||||
# through LlamaBidirectionalConfig.
|
# through LlamaBidirectionalConfig.
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class LlamaBidirectionalModel(as_embedding_model(LlamaForCausalLM)):
|
class LlamaBidirectionalModel(_LlamaBidirectionalModelBase):
|
||||||
# This class sets the correct attention type and pooling type
|
# This class sets the correct attention type and pooling type
|
||||||
# through LlamaBidirectionalConfig.
|
# through LlamaBidirectionalConfig.
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -18,7 +18,7 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
"""Inference-only LLaMA model compatible with HuggingFace weights."""
|
"""Inference-only LLaMA model compatible with HuggingFace weights."""
|
||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Callable, Iterable
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
@@ -576,6 +576,7 @@ class Llama4Model(LlamaModel):
|
|||||||
params_dict = dict(self.named_parameters())
|
params_dict = dict(self.named_parameters())
|
||||||
# The module parameters that have been loaded.
|
# The module parameters that have been loaded.
|
||||||
loaded_params: set[str] = set()
|
loaded_params: set[str] = set()
|
||||||
|
weight_loader: Callable[..., object]
|
||||||
|
|
||||||
# Iterate over all the weights and load them into module parameters.
|
# Iterate over all the weights and load them into module parameters.
|
||||||
for name, loaded_weight in weights:
|
for name, loaded_weight in weights:
|
||||||
@@ -612,9 +613,10 @@ class Llama4Model(LlamaModel):
|
|||||||
# quant config's `get_cache_scale_mapper` does not cover
|
# quant config's `get_cache_scale_mapper` does not cover
|
||||||
# (idempotent for names already renamed by the mapper).
|
# (idempotent for names already renamed by the mapper).
|
||||||
if name.endswith("scale"):
|
if name.endswith("scale"):
|
||||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||||
if name is None:
|
if remapped_name is None:
|
||||||
continue
|
continue
|
||||||
|
name = remapped_name
|
||||||
|
|
||||||
# Load the weight into the module parameter with corresponding
|
# Load the weight into the module parameter with corresponding
|
||||||
# shard id and exit the for loop and the else block.
|
# shard id and exit the for loop and the else block.
|
||||||
|
|||||||
@@ -17,6 +17,7 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -46,6 +47,19 @@ from .utils import (
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
|
||||||
|
class _EagleLlama4ForCausalLMBase(nn.Module):
|
||||||
|
def set_moe_parameters(self) -> None: ...
|
||||||
|
|
||||||
|
def permute_qk_weight_for_rotary(
|
||||||
|
self, name: str, loaded_weight: torch.Tensor
|
||||||
|
) -> tuple[str, torch.Tensor]: ...
|
||||||
|
|
||||||
|
else:
|
||||||
|
_EagleLlama4ForCausalLMBase = Llama4ForCausalLM
|
||||||
|
|
||||||
|
|
||||||
@support_torch_compile
|
@support_torch_compile
|
||||||
class LlamaModel(nn.Module):
|
class LlamaModel(nn.Module):
|
||||||
hf_to_vllm_mapper = WeightsMapper(
|
hf_to_vllm_mapper = WeightsMapper(
|
||||||
@@ -68,7 +82,9 @@ class LlamaModel(nn.Module):
|
|||||||
quant_config: QuantizationConfig | None = None,
|
quant_config: QuantizationConfig | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
speculative_config = vllm_config.speculative_config
|
||||||
|
assert speculative_config is not None
|
||||||
|
self.config = speculative_config.draft_model_config.hf_config
|
||||||
self.validate_and_update_config(start_layer_id, quant_config)
|
self.validate_and_update_config(start_layer_id, quant_config)
|
||||||
self.vocab_size = self.config.vocab_size
|
self.vocab_size = self.config.vocab_size
|
||||||
self.embed_tokens = VocabParallelEmbedding(
|
self.embed_tokens = VocabParallelEmbedding(
|
||||||
@@ -152,16 +168,18 @@ class LlamaModel(nn.Module):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class EagleLlama4ForCausalLM(Llama4ForCausalLM, SupportsMultiModalEmbeddings):
|
class EagleLlama4ForCausalLM(_EagleLlama4ForCausalLMBase, SupportsMultiModalEmbeddings):
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||||
nn.Module.__init__(self)
|
nn.Module.__init__(self)
|
||||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
speculative_config = vllm_config.speculative_config
|
||||||
|
assert speculative_config is not None
|
||||||
|
self.config = speculative_config.draft_model_config.hf_config
|
||||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||||
vllm_config.parallel_config
|
vllm_config.parallel_config
|
||||||
)
|
)
|
||||||
# draft model quantization config may differ from target model
|
# draft model quantization config may differ from target model
|
||||||
quant_config = VllmConfig.get_quantization_config(
|
quant_config = VllmConfig.get_quantization_config(
|
||||||
vllm_config.speculative_config.draft_model_config, vllm_config.load_config
|
speculative_config.draft_model_config, vllm_config.load_config
|
||||||
)
|
)
|
||||||
self.model = LlamaModel(
|
self.model = LlamaModel(
|
||||||
vllm_config=vllm_config,
|
vllm_config=vllm_config,
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -14,7 +15,10 @@ from vllm.model_executor.layers.linear import ReplicatedLinear
|
|||||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||||
from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM
|
from vllm.model_executor.models.llama import (
|
||||||
|
LlamaDecoderLayer as BaseLlamaDecoderLayer,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.models.llama import LlamaForCausalLM
|
||||||
|
|
||||||
from .utils import (
|
from .utils import (
|
||||||
AutoWeightsLoader,
|
AutoWeightsLoader,
|
||||||
@@ -27,7 +31,16 @@ from .utils import (
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class LlamaDecoderLayer(LlamaDecoderLayer):
|
if TYPE_CHECKING:
|
||||||
|
|
||||||
|
class _EagleLlamaForCausalLMBase(nn.Module):
|
||||||
|
pass
|
||||||
|
|
||||||
|
else:
|
||||||
|
_EagleLlamaForCausalLMBase = LlamaForCausalLM
|
||||||
|
|
||||||
|
|
||||||
|
class LlamaDecoderLayer(BaseLlamaDecoderLayer):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
@@ -69,7 +82,9 @@ class LlamaModel(nn.Module):
|
|||||||
start_layer_id: int = 0,
|
start_layer_id: int = 0,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
speculative_config = vllm_config.speculative_config
|
||||||
|
assert speculative_config is not None
|
||||||
|
self.config = speculative_config.draft_model_config.hf_config
|
||||||
self.vocab_size = self.config.vocab_size
|
self.vocab_size = self.config.vocab_size
|
||||||
|
|
||||||
# Get drafter's quantization config
|
# Get drafter's quantization config
|
||||||
@@ -128,10 +143,12 @@ class LlamaModel(nn.Module):
|
|||||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||||
|
|
||||||
|
|
||||||
class EagleLlamaForCausalLM(LlamaForCausalLM):
|
class EagleLlamaForCausalLM(_EagleLlamaForCausalLMBase):
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||||
nn.Module.__init__(self)
|
nn.Module.__init__(self)
|
||||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
speculative_config = vllm_config.speculative_config
|
||||||
|
assert speculative_config is not None
|
||||||
|
self.config = speculative_config.draft_model_config.hf_config
|
||||||
# Ensure draft_vocab_size is set
|
# Ensure draft_vocab_size is set
|
||||||
# default to the base vocab size when absent
|
# default to the base vocab size when absent
|
||||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -18,7 +19,14 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
|||||||
ParallelLMHead,
|
ParallelLMHead,
|
||||||
VocabParallelEmbedding,
|
VocabParallelEmbedding,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM
|
from vllm.model_executor.models.llama import (
|
||||||
|
LlamaAttention,
|
||||||
|
LlamaForCausalLM,
|
||||||
|
LlamaMLP,
|
||||||
|
)
|
||||||
|
from vllm.model_executor.models.llama import (
|
||||||
|
LlamaDecoderLayer as BaseLlamaDecoderLayer,
|
||||||
|
)
|
||||||
from vllm.multimodal.inputs import NestedTensors
|
from vllm.multimodal.inputs import NestedTensors
|
||||||
|
|
||||||
from .utils import (
|
from .utils import (
|
||||||
@@ -32,7 +40,31 @@ from .utils import (
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class LlamaDecoderLayer(LlamaDecoderLayer):
|
if TYPE_CHECKING:
|
||||||
|
|
||||||
|
class _Eagle3LlamaDecoderLayerBase(nn.Module):
|
||||||
|
hidden_size: int
|
||||||
|
self_attn: LlamaAttention
|
||||||
|
mlp: LlamaMLP
|
||||||
|
input_layernorm: RMSNorm
|
||||||
|
post_attention_layernorm: RMSNorm
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
prefix: str = "",
|
||||||
|
config: LlamaConfig | None = None,
|
||||||
|
) -> None: ...
|
||||||
|
|
||||||
|
class _Eagle3LlamaForCausalLMBase(nn.Module):
|
||||||
|
pass
|
||||||
|
|
||||||
|
else:
|
||||||
|
_Eagle3LlamaDecoderLayerBase = BaseLlamaDecoderLayer
|
||||||
|
_Eagle3LlamaForCausalLMBase = LlamaForCausalLM
|
||||||
|
|
||||||
|
|
||||||
|
class LlamaDecoderLayer(_Eagle3LlamaDecoderLayerBase):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
@@ -136,7 +168,9 @@ class LlamaModel(nn.Module):
|
|||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
speculative_config = vllm_config.speculative_config
|
||||||
|
assert speculative_config is not None
|
||||||
|
self.config = speculative_config.draft_model_config.hf_config
|
||||||
self.vocab_size = self.config.vocab_size
|
self.vocab_size = self.config.vocab_size
|
||||||
|
|
||||||
# Get drafter's quantization config
|
# Get drafter's quantization config
|
||||||
@@ -187,7 +221,7 @@ class LlamaModel(nn.Module):
|
|||||||
self.fc_input_size = target_hidden_size * self.num_aux_hidden_states
|
self.fc_input_size = target_hidden_size * self.num_aux_hidden_states
|
||||||
|
|
||||||
if self.norm_before_fc:
|
if self.norm_before_fc:
|
||||||
self.input_norm = RMSNorm(
|
self.input_norm: RMSNorm | None = RMSNorm(
|
||||||
self.fc_input_size,
|
self.fc_input_size,
|
||||||
eps=self.config.rms_norm_eps,
|
eps=self.config.rms_norm_eps,
|
||||||
)
|
)
|
||||||
@@ -269,10 +303,12 @@ class LlamaModel(nn.Module):
|
|||||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||||
|
|
||||||
|
|
||||||
class Eagle3LlamaForCausalLM(LlamaForCausalLM):
|
class Eagle3LlamaForCausalLM(_Eagle3LlamaForCausalLMBase):
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||||
nn.Module.__init__(self)
|
nn.Module.__init__(self)
|
||||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
speculative_config = vllm_config.speculative_config
|
||||||
|
assert speculative_config is not None
|
||||||
|
self.config = speculative_config.draft_model_config.hf_config
|
||||||
# Ensure draft_vocab_size is set
|
# Ensure draft_vocab_size is set
|
||||||
# default to the base vocab size when absent
|
# default to the base vocab size when absent
|
||||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||||
@@ -306,7 +342,7 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM):
|
|||||||
requires_grad=False,
|
requires_grad=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.use_parallel_drafting = vllm_config.speculative_config.parallel_drafting
|
self.use_parallel_drafting = speculative_config.parallel_drafting
|
||||||
|
|
||||||
if self.use_parallel_drafting:
|
if self.use_parallel_drafting:
|
||||||
self.register_buffer(
|
self.register_buffer(
|
||||||
@@ -365,6 +401,7 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM):
|
|||||||
# combine multiple auxiliary hidden states returned by eagle3
|
# combine multiple auxiliary hidden states returned by eagle3
|
||||||
|
|
||||||
if self.model.norm_before_fc:
|
if self.model.norm_before_fc:
|
||||||
|
assert self.model.input_norm is not None
|
||||||
hidden_states = self.model.input_norm(hidden_states)
|
hidden_states = self.model.input_norm(hidden_states)
|
||||||
|
|
||||||
# `norm_before_fc` adds a single RMSNorm before the FC layer, whereas `fc_norm`
|
# `norm_before_fc` adds a single RMSNorm before the FC layer, whereas `fc_norm`
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar
|
from typing import Annotated, Literal, Protocol, TypeAlias, TypeVar
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -159,14 +159,14 @@ class LlavaMultiModalProjector(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class LlavaLikeConfig(Protocol):
|
class LlavaLikeConfig(Protocol):
|
||||||
vision_config: Final[PretrainedConfig]
|
vision_config: PretrainedConfig
|
||||||
image_token_index: Final[int]
|
image_token_index: int
|
||||||
vision_feature_select_strategy: Final[str]
|
vision_feature_select_strategy: str
|
||||||
vision_feature_layer: Final[int | list[int]]
|
vision_feature_layer: int | list[int]
|
||||||
|
|
||||||
|
|
||||||
class LlavaLikeProcessor(Protocol):
|
class LlavaLikeProcessor(Protocol):
|
||||||
image_token: Final[str]
|
image_token: str
|
||||||
|
|
||||||
|
|
||||||
class BaseLlavaProcessingInfo(BaseProcessingInfo):
|
class BaseLlavaProcessingInfo(BaseProcessingInfo):
|
||||||
@@ -286,6 +286,7 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
if isinstance(images, ImageEmbeddingItems):
|
if isinstance(images, ImageEmbeddingItems):
|
||||||
num_image_tokens = images.get_feature_size(item_idx)
|
num_image_tokens = images.get_feature_size(item_idx)
|
||||||
else:
|
else:
|
||||||
|
assert isinstance(images, ImageProcessorItems)
|
||||||
image_size = images.get_image_size(item_idx)
|
image_size = images.get_image_size(item_idx)
|
||||||
num_image_tokens = self.info.get_num_image_tokens(
|
num_image_tokens = self.info.get_num_image_tokens(
|
||||||
image_width=image_size.width,
|
image_width=image_size.width,
|
||||||
@@ -372,6 +373,7 @@ class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingIn
|
|||||||
image_end_id = vocab[processor.image_end_token]
|
image_end_id = vocab[processor.image_end_token]
|
||||||
|
|
||||||
assert isinstance(hf_config.vision_config, PixtralVisionConfig)
|
assert isinstance(hf_config.vision_config, PixtralVisionConfig)
|
||||||
|
assert isinstance(hf_config, LlavaConfig)
|
||||||
encoder_info = PixtralHFEncoderInfo(hf_config)
|
encoder_info = PixtralHFEncoderInfo(hf_config)
|
||||||
|
|
||||||
def get_replacement(item_idx: int):
|
def get_replacement(item_idx: int):
|
||||||
@@ -642,7 +644,7 @@ class LlavaForConditionalGeneration(
|
|||||||
self,
|
self,
|
||||||
image_input: LlavaImageInputs,
|
image_input: LlavaImageInputs,
|
||||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||||
if image_input["type"] == "image_embeds":
|
if isinstance(image_input, LlavaImageEmbeddingInputs):
|
||||||
return image_input["data"]
|
return image_input["data"]
|
||||||
|
|
||||||
image_features = self._process_image_pixels(image_input)
|
image_features = self._process_image_pixels(image_input)
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
from collections.abc import Iterable, Mapping
|
from collections.abc import Iterable, Mapping
|
||||||
from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar
|
from typing import Annotated, Literal, Protocol, TypeAlias, TypeVar
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -88,7 +88,7 @@ LlavaNextImageInputs: TypeAlias = (
|
|||||||
|
|
||||||
|
|
||||||
class LlavaNextLikeConfig(LlavaLikeConfig, Protocol):
|
class LlavaNextLikeConfig(LlavaLikeConfig, Protocol):
|
||||||
image_grid_pinpoints: Final[list[list[int]]]
|
image_grid_pinpoints: list[list[int]]
|
||||||
|
|
||||||
|
|
||||||
class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):
|
class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):
|
||||||
@@ -483,7 +483,7 @@ class LlavaNextForConditionalGeneration(
|
|||||||
self,
|
self,
|
||||||
image_input: LlavaNextImageInputs,
|
image_input: LlavaNextImageInputs,
|
||||||
) -> torch.Tensor | list[torch.Tensor]:
|
) -> torch.Tensor | list[torch.Tensor]:
|
||||||
if image_input["type"] == "image_embeds":
|
if isinstance(image_input, LlavaNextImageEmbeddingInputs):
|
||||||
return image_input["data"]
|
return image_input["data"]
|
||||||
|
|
||||||
patch_embeddings = self._process_image_pixels(image_input)
|
patch_embeddings = self._process_image_pixels(image_input)
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import torch.nn as nn
|
|||||||
from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProcessor
|
from transformers import BatchFeature, LlavaNextVideoConfig, LlavaNextVideoProcessor
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||||
from vllm.inputs import MultiModalDataDict
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.model_executor.models.clip import CLIPVisionModel
|
from vllm.model_executor.models.clip import CLIPVisionModel
|
||||||
@@ -181,6 +181,7 @@ class LlavaNextVideoDummyInputsBuilder(
|
|||||||
)
|
)
|
||||||
|
|
||||||
video_overrides = mm_options.get("video")
|
video_overrides = mm_options.get("video")
|
||||||
|
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"video": self._get_dummy_videos(
|
"video": self._get_dummy_videos(
|
||||||
@@ -220,6 +221,7 @@ class LlavaNextVideoMultiModalProcessor(
|
|||||||
if isinstance(videos, VideoEmbeddingItems):
|
if isinstance(videos, VideoEmbeddingItems):
|
||||||
num_video_tokens = videos.get_feature_size(item_idx)
|
num_video_tokens = videos.get_feature_size(item_idx)
|
||||||
else:
|
else:
|
||||||
|
assert isinstance(videos, VideoProcessorItems)
|
||||||
image_size = videos.get_frame_size(item_idx)
|
image_size = videos.get_frame_size(item_idx)
|
||||||
num_video_tokens = self.info.get_num_video_tokens(
|
num_video_tokens = self.info.get_num_video_tokens(
|
||||||
image_width=image_size.width,
|
image_width=image_size.width,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
import math
|
import math
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from typing import Annotated, Final, Literal, Protocol, TypeAlias
|
from typing import Annotated, Literal, Protocol, TypeAlias, TypedDict
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -14,7 +14,7 @@ from transformers.models.llava_onevision.modeling_llava_onevision import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||||
from vllm.inputs import MultiModalDataDict
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.activation import get_act_fn
|
from vllm.model_executor.layers.activation import get_act_fn
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
@@ -123,7 +123,12 @@ LlavaOnevisionMultiInputs: TypeAlias = (
|
|||||||
|
|
||||||
|
|
||||||
class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol):
|
class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol):
|
||||||
video_token_index: Final[int]
|
video_token_index: int
|
||||||
|
|
||||||
|
|
||||||
|
class LlavaOnevisionInputsByModality(TypedDict, total=False):
|
||||||
|
image: LlavaOnevisionImageInputs | None
|
||||||
|
video: LlavaOnevisionVideoPixelInputs | None
|
||||||
|
|
||||||
|
|
||||||
class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
|
class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
|
||||||
@@ -288,6 +293,7 @@ class LlavaOnevisionDummyInputsBuilder(
|
|||||||
|
|
||||||
image_overrides = mm_options.get("image")
|
image_overrides = mm_options.get("image")
|
||||||
video_overrides = mm_options.get("video")
|
video_overrides = mm_options.get("video")
|
||||||
|
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"image": self._get_dummy_images(
|
"image": self._get_dummy_images(
|
||||||
@@ -417,6 +423,7 @@ class LlavaOnevisionMultiModalProcessor(
|
|||||||
if isinstance(videos, VideoEmbeddingItems):
|
if isinstance(videos, VideoEmbeddingItems):
|
||||||
num_video_tokens = videos.get_feature_size(item_idx)
|
num_video_tokens = videos.get_feature_size(item_idx)
|
||||||
else:
|
else:
|
||||||
|
assert isinstance(videos, VideoProcessorItems)
|
||||||
image_size = videos.get_frame_size(item_idx)
|
image_size = videos.get_frame_size(item_idx)
|
||||||
num_video_tokens = self.info.get_num_video_tokens(
|
num_video_tokens = self.info.get_num_video_tokens(
|
||||||
image_width=image_size.width,
|
image_width=image_size.width,
|
||||||
@@ -570,8 +577,10 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
def _parse_and_validate_multimodal_inputs(
|
||||||
mm_input_by_modality = {}
|
self, **kwargs: object
|
||||||
|
) -> LlavaOnevisionInputsByModality:
|
||||||
|
mm_input_by_modality = LlavaOnevisionInputsByModality()
|
||||||
|
|
||||||
# Preserve the order of modalities if there are multiple of them
|
# Preserve the order of modalities if there are multiple of them
|
||||||
# from the order of kwargs.
|
# from the order of kwargs.
|
||||||
@@ -749,7 +758,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
|||||||
self,
|
self,
|
||||||
image_input: LlavaOnevisionImageInputs,
|
image_input: LlavaOnevisionImageInputs,
|
||||||
) -> torch.Tensor | list[torch.Tensor]:
|
) -> torch.Tensor | list[torch.Tensor]:
|
||||||
if image_input["type"] == "image_embeds":
|
if isinstance(image_input, LlavaOnevisionImageEmbeddingInputs):
|
||||||
return image_input["data"]
|
return image_input["data"]
|
||||||
|
|
||||||
patch_embeddings = self._process_image_pixels(image_input)
|
patch_embeddings = self._process_image_pixels(image_input)
|
||||||
@@ -860,12 +869,15 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
|||||||
# NOTE: It is important to iterate over the keys in this dictionary
|
# NOTE: It is important to iterate over the keys in this dictionary
|
||||||
# to preserve the order of the modalities.
|
# to preserve the order of the modalities.
|
||||||
for modality in mm_input_by_modality:
|
for modality in mm_input_by_modality:
|
||||||
multimodal_input = mm_input_by_modality[modality]
|
|
||||||
if modality == "image":
|
if modality == "image":
|
||||||
image_embeddings = self._process_image_input(multimodal_input)
|
image_input = mm_input_by_modality["image"]
|
||||||
|
assert image_input is not None
|
||||||
|
image_embeddings = self._process_image_input(image_input)
|
||||||
multimodal_embeddings += tuple(image_embeddings)
|
multimodal_embeddings += tuple(image_embeddings)
|
||||||
if modality == "video":
|
if modality == "video":
|
||||||
video_embeddings = self._process_video_pixels(multimodal_input)
|
video_input = mm_input_by_modality["video"]
|
||||||
|
assert video_input is not None
|
||||||
|
video_embeddings = self._process_video_pixels(video_input)
|
||||||
multimodal_embeddings += tuple(video_embeddings)
|
multimodal_embeddings += tuple(video_embeddings)
|
||||||
|
|
||||||
return multimodal_embeddings
|
return multimodal_embeddings
|
||||||
|
|||||||
@@ -24,12 +24,15 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import importlib
|
import importlib
|
||||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
from collections.abc import Callable, Iterable, Mapping, Sequence, Sized
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from typing import (
|
from typing import (
|
||||||
Annotated,
|
Annotated,
|
||||||
Any,
|
Any,
|
||||||
Literal,
|
Literal,
|
||||||
|
SupportsIndex,
|
||||||
|
SupportsInt,
|
||||||
|
TypedDict,
|
||||||
)
|
)
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -45,6 +48,7 @@ from transformers.dynamic_module_utils import (
|
|||||||
)
|
)
|
||||||
from transformers.models.qwen2_vl import Qwen2VLImageProcessor
|
from transformers.models.qwen2_vl import Qwen2VLImageProcessor
|
||||||
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
|
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
|
||||||
|
from typing_extensions import Buffer
|
||||||
|
|
||||||
from vllm.compilation.decorators import (
|
from vllm.compilation.decorators import (
|
||||||
should_torch_compile_mm_encoder,
|
should_torch_compile_mm_encoder,
|
||||||
@@ -730,6 +734,11 @@ LlavaOnevision2ImageInputs = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class LlavaOnevision2InputsByModality(TypedDict, total=False):
|
||||||
|
images: LlavaOnevision2ImageInputs | None
|
||||||
|
videos: LlavaOnevision2VideoPixelInputs | None
|
||||||
|
|
||||||
|
|
||||||
class LlavaOnevision2VisionRotaryEmbedding(nn.Module):
|
class LlavaOnevision2VisionRotaryEmbedding(nn.Module):
|
||||||
"""3-D rotary frequency constructor with 4:6:6 (T:H:W) split.
|
"""3-D rotary frequency constructor with 4:6:6 (T:H:W) split.
|
||||||
|
|
||||||
@@ -1468,7 +1477,7 @@ class LlavaOnevision2DummyInputsBuilder(
|
|||||||
n_img = mm_counts.get("image", 0)
|
n_img = mm_counts.get("image", 0)
|
||||||
n_vid = mm_counts.get("video", 0)
|
n_vid = mm_counts.get("video", 0)
|
||||||
w, h = self.info.get_image_size_with_most_features()
|
w, h = self.info.get_image_size_with_most_features()
|
||||||
out: MultiModalDataDict = {}
|
out: dict[str, object] = {}
|
||||||
if n_img:
|
if n_img:
|
||||||
out["image"] = self._get_dummy_images(width=w, height=h, num_images=n_img)
|
out["image"] = self._get_dummy_images(width=w, height=h, num_images=n_img)
|
||||||
if n_vid:
|
if n_vid:
|
||||||
@@ -1586,6 +1595,8 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
# array, or tensor, and ``and <array>`` would raise on the ambiguous
|
# array, or tensor, and ``and <array>`` would raise on the ambiguous
|
||||||
# truth value of a multi-element array.
|
# truth value of a multi-element array.
|
||||||
_videos = mm_data.get("videos")
|
_videos = mm_data.get("videos")
|
||||||
|
if _videos is not None and not isinstance(_videos, Sized):
|
||||||
|
raise TypeError(f"object of type '{type(_videos).__name__}' has no len()")
|
||||||
videos_present = _videos is not None and len(_videos) > 0
|
videos_present = _videos is not None and len(_videos) > 0
|
||||||
|
|
||||||
codec_video_paths = (
|
codec_video_paths = (
|
||||||
@@ -1666,15 +1677,24 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
# provided by the backend's ``compute_frames_index_to_sample``; SSRF /
|
# provided by the backend's ``compute_frames_index_to_sample``; SSRF /
|
||||||
# local-file gating is enforced by the connector before decoding.
|
# local-file gating is enforced by the connector before decoding.
|
||||||
if videos_present:
|
if videos_present:
|
||||||
timestamp_decimals = int(
|
timestamp_decimals = hf_processor_mm_kwargs.get(
|
||||||
hf_processor_mm_kwargs.get(
|
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
|
||||||
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
if not isinstance(
|
||||||
|
timestamp_decimals, (str, Buffer, SupportsInt, SupportsIndex)
|
||||||
|
):
|
||||||
|
raise TypeError(
|
||||||
|
"int() argument must be a string, a bytes-like object "
|
||||||
|
f"or a real number, not '{type(timestamp_decimals).__name__}'"
|
||||||
|
)
|
||||||
|
timestamp_decimals = int(timestamp_decimals)
|
||||||
|
|
||||||
per_video_frames: list[list[Image.Image]] = []
|
per_video_frames: list[list[Image.Image]] = []
|
||||||
per_video_timestamps: list[list[float]] = []
|
per_video_timestamps: list[list[float]] = []
|
||||||
for item in mm_data["videos"]:
|
videos = mm_data["videos"]
|
||||||
|
if not isinstance(videos, Iterable):
|
||||||
|
raise TypeError(f"Unsupported videos input: {type(videos)}")
|
||||||
|
for item in videos:
|
||||||
pil_frames, timestamps = _frame_video_to_pil_and_timestamps(item)
|
pil_frames, timestamps = _frame_video_to_pil_and_timestamps(item)
|
||||||
per_video_frames.append(pil_frames)
|
per_video_frames.append(pil_frames)
|
||||||
per_video_timestamps.append(timestamps)
|
per_video_timestamps.append(timestamps)
|
||||||
@@ -1902,18 +1922,21 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
def get_image_replacement(item_idx: int):
|
def get_image_replacement(item_idx: int):
|
||||||
out_item = out_mm_kwargs["image"][item_idx]
|
out_item = out_mm_kwargs["image"][item_idx]
|
||||||
grid_thw = out_item["image_grid_thw"].data
|
grid_thw = out_item["image_grid_thw"].data
|
||||||
|
assert isinstance(grid_thw, torch.Tensor)
|
||||||
n = int(grid_thw.prod(-1).sum()) // merge_length
|
n = int(grid_thw.prod(-1).sum()) // merge_length
|
||||||
return [image_pad_id] * n
|
return [image_pad_id] * n
|
||||||
|
|
||||||
def get_video_replacement(item_idx: int):
|
def get_video_replacement(item_idx: int):
|
||||||
out_item = out_mm_kwargs["video"][item_idx]
|
out_item = out_mm_kwargs["video"][item_idx]
|
||||||
grid_thw = out_item["video_grid_thw"].data
|
grid_thw = out_item["video_grid_thw"].data
|
||||||
|
assert isinstance(grid_thw, torch.Tensor)
|
||||||
is_codec_field = out_item.get("video_is_codec")
|
is_codec_field = out_item.get("video_is_codec")
|
||||||
is_codec = (
|
if is_codec_field is not None:
|
||||||
bool(int(is_codec_field.data.item()))
|
is_codec_data = is_codec_field.data
|
||||||
if is_codec_field is not None
|
assert isinstance(is_codec_data, torch.Tensor)
|
||||||
else False
|
is_codec = bool(int(is_codec_data.item()))
|
||||||
)
|
else:
|
||||||
|
is_codec = False
|
||||||
tokens: list[int] = []
|
tokens: list[int] = []
|
||||||
|
|
||||||
if is_codec:
|
if is_codec:
|
||||||
@@ -1924,6 +1947,8 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
# run, emit ``<sec seconds><|vision_start|><pad*N><|vision_end|>\n``.
|
# run, emit ``<sec seconds><|vision_start|><pad*N><|vision_end|>\n``.
|
||||||
patch_positions = out_item["patch_positions_videos"].data
|
patch_positions = out_item["patch_positions_videos"].data
|
||||||
fps_t = out_item["codec_fps"].data
|
fps_t = out_item["codec_fps"].data
|
||||||
|
assert isinstance(patch_positions, torch.Tensor)
|
||||||
|
assert isinstance(fps_t, torch.Tensor)
|
||||||
fps = float(int(fps_t.item())) / 1_000_000_000.0
|
fps = float(int(fps_t.item())) / 1_000_000_000.0
|
||||||
runs = _codec_timestamp_runs(
|
runs = _codec_timestamp_runs(
|
||||||
patch_positions, fps, image_processor.merge_size
|
patch_positions, fps, image_processor.merge_size
|
||||||
@@ -1938,6 +1963,7 @@ class LlavaOnevision2MultiModalProcessor(
|
|||||||
tokens.extend(newline_ids)
|
tokens.extend(newline_ids)
|
||||||
else:
|
else:
|
||||||
timestamps = out_item["frame_timestamps"].data
|
timestamps = out_item["frame_timestamps"].data
|
||||||
|
assert isinstance(timestamps, torch.Tensor)
|
||||||
T_total = int(grid_thw.shape[0])
|
T_total = int(grid_thw.shape[0])
|
||||||
for t in range(T_total):
|
for t in range(T_total):
|
||||||
sec = float(timestamps[t].item())
|
sec = float(timestamps[t].item())
|
||||||
@@ -2121,7 +2147,7 @@ class LlavaOnevision2ForConditionalGeneration(
|
|||||||
grid_thw = image_input["image_grid_thw"]
|
grid_thw = image_input["image_grid_thw"]
|
||||||
assert grid_thw.ndim == 2
|
assert grid_thw.ndim == 2
|
||||||
|
|
||||||
if image_input["type"] == "image_embeds":
|
if isinstance(image_input, LlavaOnevision2ImageEmbeddingInputs):
|
||||||
image_embeds = image_input["image_embeds"]
|
image_embeds = image_input["image_embeds"]
|
||||||
else:
|
else:
|
||||||
image_embeds = self.visual(
|
image_embeds = self.visual(
|
||||||
@@ -2166,8 +2192,13 @@ class LlavaOnevision2ForConditionalGeneration(
|
|||||||
)
|
)
|
||||||
if isinstance(video_num_frames, list):
|
if isinstance(video_num_frames, list):
|
||||||
video_num_frames = torch.cat([v.flatten() for v in video_num_frames])
|
video_num_frames = torch.cat([v.flatten() for v in video_num_frames])
|
||||||
else:
|
elif isinstance(video_num_frames, torch.Tensor):
|
||||||
video_num_frames = video_num_frames.flatten()
|
video_num_frames = video_num_frames.flatten()
|
||||||
|
else:
|
||||||
|
raise TypeError(
|
||||||
|
"video_num_frames must be a tensor or a list of tensors, "
|
||||||
|
f"got {type(video_num_frames)}"
|
||||||
|
)
|
||||||
return LlavaOnevision2VideoPixelInputs(
|
return LlavaOnevision2VideoPixelInputs(
|
||||||
type="pixel_values_videos",
|
type="pixel_values_videos",
|
||||||
pixel_values_videos=pixel_values_videos,
|
pixel_values_videos=pixel_values_videos,
|
||||||
@@ -2202,8 +2233,10 @@ class LlavaOnevision2ForConditionalGeneration(
|
|||||||
cursor += n
|
cursor += n
|
||||||
return video_embeds.split(sizes)
|
return video_embeds.split(sizes)
|
||||||
|
|
||||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
def _parse_and_validate_multimodal_inputs(
|
||||||
modalities = {}
|
self, **kwargs: object
|
||||||
|
) -> LlavaOnevision2InputsByModality:
|
||||||
|
modalities = LlavaOnevision2InputsByModality()
|
||||||
for key in kwargs:
|
for key in kwargs:
|
||||||
if key in ("pixel_values", "image_embeds") and "images" not in modalities:
|
if key in ("pixel_values", "image_embeds") and "images" not in modalities:
|
||||||
modalities["images"] = self._parse_and_validate_image_input(**kwargs)
|
modalities["images"] = self._parse_and_validate_image_input(**kwargs)
|
||||||
@@ -2221,9 +2254,13 @@ class LlavaOnevision2ForConditionalGeneration(
|
|||||||
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
|
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
|
||||||
for modality in modalities:
|
for modality in modalities:
|
||||||
if modality == "images":
|
if modality == "images":
|
||||||
multimodal_embeddings += self._process_image_input(modalities["images"])
|
image_input = modalities["images"]
|
||||||
|
assert image_input is not None
|
||||||
|
multimodal_embeddings += self._process_image_input(image_input)
|
||||||
elif modality == "videos":
|
elif modality == "videos":
|
||||||
multimodal_embeddings += self._process_video_input(modalities["videos"])
|
video_input = modalities["videos"]
|
||||||
|
assert video_input is not None
|
||||||
|
multimodal_embeddings += self._process_video_input(video_input)
|
||||||
return multimodal_embeddings
|
return multimodal_embeddings
|
||||||
|
|
||||||
def get_input_embeddings(
|
def get_input_embeddings(
|
||||||
|
|||||||
@@ -83,6 +83,8 @@ logger = init_logger(__name__)
|
|||||||
class FlashConfig(PretrainedConfig):
|
class FlashConfig(PretrainedConfig):
|
||||||
"""Flash model configuration."""
|
"""Flash model configuration."""
|
||||||
|
|
||||||
|
moe_intermediate_size: int
|
||||||
|
|
||||||
model_type = "longcat_flash"
|
model_type = "longcat_flash"
|
||||||
keys_to_ignore_at_inference = ["past_key_values"]
|
keys_to_ignore_at_inference = ["past_key_values"]
|
||||||
|
|
||||||
@@ -604,7 +606,7 @@ class FlashModel(nn.Module):
|
|||||||
else:
|
else:
|
||||||
is_expert_weight = False
|
is_expert_weight = False
|
||||||
for mapping in expert_params_mapping:
|
for mapping in expert_params_mapping:
|
||||||
param_name, weight_name, expert_id, shard_id = mapping
|
param_name, weight_name, expert_id, expert_shard_id = mapping
|
||||||
if weight_name not in name:
|
if weight_name not in name:
|
||||||
continue
|
continue
|
||||||
is_expert_weight = True
|
is_expert_weight = True
|
||||||
@@ -627,7 +629,7 @@ class FlashModel(nn.Module):
|
|||||||
param,
|
param,
|
||||||
loaded_weight,
|
loaded_weight,
|
||||||
name_mapped,
|
name_mapped,
|
||||||
shard_id=shard_id,
|
shard_id=expert_shard_id,
|
||||||
expert_id=expert_id,
|
expert_id=expert_id,
|
||||||
return_success=True,
|
return_success=True,
|
||||||
)
|
)
|
||||||
@@ -664,11 +666,14 @@ class FlashModel(nn.Module):
|
|||||||
if isinstance(self.layers[layer_id], PPMissingLayer):
|
if isinstance(self.layers[layer_id], PPMissingLayer):
|
||||||
continue
|
continue
|
||||||
self_attn = self.layers[layer_id].self_attn[i]
|
self_attn = self.layers[layer_id].self_attn[i]
|
||||||
if hasattr(
|
if (
|
||||||
self.quant_config, "weight_block_size"
|
self.quant_config is not None
|
||||||
) and self_attn.kv_b_proj.weight.dtype in (
|
and hasattr(self.quant_config, "weight_block_size")
|
||||||
torch.float8_e4m3fn,
|
and self_attn.kv_b_proj.weight.dtype
|
||||||
torch.float8_e4m3fnuz,
|
in (
|
||||||
|
torch.float8_e4m3fn,
|
||||||
|
torch.float8_e4m3fnuz,
|
||||||
|
)
|
||||||
):
|
):
|
||||||
weight_block_size = self.quant_config.weight_block_size
|
weight_block_size = self.quant_config.weight_block_size
|
||||||
if weight_block_size is not None:
|
if weight_block_size is not None:
|
||||||
|
|||||||
@@ -266,11 +266,14 @@ class LongCatFlashMTP(nn.Module):
|
|||||||
loaded_params.add(name)
|
loaded_params.add(name)
|
||||||
spec_layer_id = self.config.num_hidden_layers * 2
|
spec_layer_id = self.config.num_hidden_layers * 2
|
||||||
self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn
|
self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn
|
||||||
if hasattr(
|
if (
|
||||||
self.quant_config, "weight_block_size"
|
self.quant_config is not None
|
||||||
) and self_attn.kv_b_proj.weight.dtype in (
|
and hasattr(self.quant_config, "weight_block_size")
|
||||||
torch.float8_e4m3fn,
|
and self_attn.kv_b_proj.weight.dtype
|
||||||
torch.float8_e4m3fnuz,
|
in (
|
||||||
|
torch.float8_e4m3fn,
|
||||||
|
torch.float8_e4m3fnuz,
|
||||||
|
)
|
||||||
):
|
):
|
||||||
weight_block_size = self.quant_config.weight_block_size
|
weight_block_size = self.quant_config.weight_block_size
|
||||||
if weight_block_size is not None:
|
if weight_block_size is not None:
|
||||||
|
|||||||
@@ -178,6 +178,7 @@ class FlashNgramModel(FlashModel):
|
|||||||
if num_layers is not None and hf.num_hidden_layers != num_layers:
|
if num_layers is not None and hf.num_hidden_layers != num_layers:
|
||||||
hf.num_hidden_layers = num_layers
|
hf.num_hidden_layers = num_layers
|
||||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||||
|
self.ngram_embeddings: NgramEmbedding | None
|
||||||
if get_pp_group().is_first_rank and uses_ngram_embedding(self.config):
|
if get_pp_group().is_first_rank and uses_ngram_embedding(self.config):
|
||||||
self.ngram_embeddings = NgramEmbedding(self.config, self.embed_tokens)
|
self.ngram_embeddings = NgramEmbedding(self.config, self.embed_tokens)
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -217,7 +217,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
|
|||||||
def get_vision_encoder_info(
|
def get_vision_encoder_info(
|
||||||
self,
|
self,
|
||||||
mm_processor_kwargs: Mapping[str, object] | None = None,
|
mm_processor_kwargs: Mapping[str, object] | None = None,
|
||||||
) -> Mistral3HFEncoderInfo:
|
) -> PixtralHFEncoderInfo:
|
||||||
processor = self.get_hf_processor()
|
processor = self.get_hf_processor()
|
||||||
size = processor.image_processor.size
|
size = processor.image_processor.size
|
||||||
merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})
|
merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})
|
||||||
|
|||||||
Reference in New Issue
Block a user