[Mypy] Fix mypy typing for model interfaces and H/I models (#54079)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Wentao Ye
parent
74850f9f66
commit
21fa2c5a2a
@@ -112,8 +112,6 @@ SEPARATE_GROUPS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
EXCLUDE = [
|
EXCLUDE = [
|
||||||
r"vllm/model_executor/models/[hH]",
|
|
||||||
r"vllm/model_executor/models/[iI]",
|
|
||||||
r"vllm/model_executor/models/[jJ]",
|
r"vllm/model_executor/models/[jJ]",
|
||||||
r"vllm/model_executor/models/[kK]",
|
r"vllm/model_executor/models/[kK]",
|
||||||
r"vllm/model_executor/models/[lL]",
|
r"vllm/model_executor/models/[lL]",
|
||||||
|
|||||||
@@ -430,7 +430,7 @@ class ChatGLMBaseModel(nn.Module):
|
|||||||
|
|
||||||
|
|
||||||
class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP, SupportsQuant):
|
class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping: dict[str, list[str]] = {
|
||||||
"query_key_value": ["query_key_value"],
|
"query_key_value": ["query_key_value"],
|
||||||
"dense_h_to_4h": ["dense_h_to_4h"],
|
"dense_h_to_4h": ["dense_h_to_4h"],
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -417,7 +417,7 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
|||||||
orig_to_new_substr={"_quantizer.": None},
|
orig_to_new_substr={"_quantizer.": None},
|
||||||
orig_to_new_prefix={"lm_head": None},
|
orig_to_new_prefix={"lm_head": None},
|
||||||
)
|
)
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping: dict[str, list[str]] = {
|
||||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -339,7 +339,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
|||||||
".up_proj": (".gate_up_proj", 1),
|
".up_proj": (".gate_up_proj", 1),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping: dict[str, list[str]] = {
|
||||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1019,7 +1019,7 @@ class Gemma4ForConditionalGeneration(
|
|||||||
mm_prefix_clamp_sliding_window: bool = True
|
mm_prefix_clamp_sliding_window: bool = True
|
||||||
supports_tower_connector_lora = True
|
supports_tower_connector_lora = True
|
||||||
|
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping: dict[str, list[str]] = {
|
||||||
"qkv_proj": [
|
"qkv_proj": [
|
||||||
"q_proj",
|
"q_proj",
|
||||||
"k_proj",
|
"k_proj",
|
||||||
|
|||||||
@@ -395,7 +395,7 @@ class GraniteModel(nn.Module):
|
|||||||
class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||||
hf_to_vllm_mapper = GraniteModel.hf_to_vllm_mapper
|
hf_to_vllm_mapper = GraniteModel.hf_to_vllm_mapper
|
||||||
# LoRA specific attributes
|
# LoRA specific attributes
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping: dict[str, list[str]] = {
|
||||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -587,7 +587,7 @@ class GraniteMoeHybridForCausalLM(
|
|||||||
SupportsQuant,
|
SupportsQuant,
|
||||||
SupportsMambaPrefixCaching,
|
SupportsMambaPrefixCaching,
|
||||||
):
|
):
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping: dict[str, list[str]] = {
|
||||||
"qkv_proj": [
|
"qkv_proj": [
|
||||||
"q_proj",
|
"q_proj",
|
||||||
"k_proj",
|
"k_proj",
|
||||||
|
|||||||
@@ -114,6 +114,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
|
|||||||
if isinstance(images, ImageEmbeddingItems):
|
if isinstance(images, ImageEmbeddingItems):
|
||||||
feature_size = images.get_feature_size(item_idx)
|
feature_size = images.get_feature_size(item_idx)
|
||||||
else:
|
else:
|
||||||
|
assert isinstance(images, ImageProcessorItems)
|
||||||
image_size = images.get_image_size(item_idx)
|
image_size = images.get_image_size(item_idx)
|
||||||
feature_size = self.info.get_num_image_tokens(
|
feature_size = self.info.get_num_image_tokens(
|
||||||
image_width=image_size.width,
|
image_width=image_size.width,
|
||||||
|
|||||||
@@ -248,7 +248,7 @@ class HrmTextDecoderLayer(nn.Module):
|
|||||||
self,
|
self,
|
||||||
config: PretrainedConfig,
|
config: PretrainedConfig,
|
||||||
layer_idx_in_stack: int,
|
layer_idx_in_stack: int,
|
||||||
stack_kind: str,
|
stack_kind: Literal["L", "H"],
|
||||||
cache_config: CacheConfig | None = None,
|
cache_config: CacheConfig | None = None,
|
||||||
quant_config: QuantizationConfig | None = None,
|
quant_config: QuantizationConfig | None = None,
|
||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
@@ -312,7 +312,7 @@ class HrmTextStack(nn.Module):
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
config: PretrainedConfig,
|
config: PretrainedConfig,
|
||||||
stack_kind: str,
|
stack_kind: Literal["L", "H"],
|
||||||
cache_config: CacheConfig | None = None,
|
cache_config: CacheConfig | None = None,
|
||||||
quant_config: QuantizationConfig | None = None,
|
quant_config: QuantizationConfig | None = None,
|
||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
|
|||||||
@@ -161,6 +161,7 @@ class HYV3MoEFused(nn.Module):
|
|||||||
prefix=f"{prefix}.gate",
|
prefix=f"{prefix}.gate",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.shared_mlp: HYV3FeedForward | None
|
||||||
if config.num_shared_experts > 0:
|
if config.num_shared_experts > 0:
|
||||||
self.shared_mlp = HYV3FeedForward(
|
self.shared_mlp = HYV3FeedForward(
|
||||||
hidden_size=config.hidden_size,
|
hidden_size=config.hidden_size,
|
||||||
@@ -467,28 +468,27 @@ class HYV3Model(nn.Module, MixtureOfExperts):
|
|||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.num_expert_groups = 1
|
self.num_expert_groups = 1
|
||||||
self.moe_layers = []
|
self.moe_layers = []
|
||||||
example_layer = None
|
example_layer: HYV3MoEFused | None = None
|
||||||
for layer in self.layers:
|
for layer in self.layers:
|
||||||
if isinstance(layer, PPMissingLayer):
|
if isinstance(layer, PPMissingLayer):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
assert isinstance(layer, HYV3DecoderLayer)
|
assert isinstance(layer, HYV3DecoderLayer)
|
||||||
if layer.block_type == "moe":
|
if layer.block_type == "moe":
|
||||||
|
assert isinstance(layer.mlp, HYV3MoEFused)
|
||||||
example_layer = layer.mlp
|
example_layer = layer.mlp
|
||||||
self.moe_layers.append(layer.mlp.experts)
|
self.moe_layers.append(example_layer.experts)
|
||||||
|
|
||||||
if example_layer is None:
|
if example_layer is None:
|
||||||
self.num_moe_layers = 0
|
self.num_moe_layers = 0
|
||||||
raise RuntimeError("No MoE layer found in model.layers.")
|
raise RuntimeError("No MoE layer found in model.layers.")
|
||||||
|
|
||||||
self.num_moe_layers = len(self.moe_layers)
|
self.num_moe_layers = len(self.moe_layers)
|
||||||
self.num_logical_experts = getattr(example_layer, "n_logical_experts", None)
|
self.num_logical_experts = example_layer.n_logical_experts
|
||||||
self.num_physical_experts = getattr(example_layer, "n_physical_experts", None)
|
self.num_physical_experts = example_layer.n_physical_experts
|
||||||
self.num_local_physical_experts = getattr(
|
self.num_local_physical_experts = example_layer.n_local_physical_experts
|
||||||
example_layer, "n_local_physical_experts", None
|
self.num_routed_experts = example_layer.n_routed_experts
|
||||||
)
|
self.num_redundant_experts = example_layer.n_redundant_experts
|
||||||
self.num_routed_experts = getattr(example_layer, "n_routed_experts", None)
|
|
||||||
self.num_redundant_experts = getattr(example_layer, "n_redundant_experts", None)
|
|
||||||
|
|
||||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||||
return self.embed_tokens(input_ids)
|
return self.embed_tokens(input_ids)
|
||||||
@@ -572,9 +572,10 @@ class HYV3Model(nn.Module, MixtureOfExperts):
|
|||||||
continue
|
continue
|
||||||
if "scale" in name:
|
if "scale" in name:
|
||||||
# Remapping the name of FP8 kv-scale.
|
# Remapping the name of FP8 kv-scale.
|
||||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||||
if name is None:
|
if remapped_name is None:
|
||||||
continue
|
continue
|
||||||
|
name = remapped_name
|
||||||
is_found = False
|
is_found = False
|
||||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||||
if weight_name not in name:
|
if weight_name not in name:
|
||||||
|
|||||||
@@ -24,7 +24,7 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
"""Inference-only HY V3 MTP model compatible with HuggingFace weights."""
|
"""Inference-only HY V3 MTP model compatible with HuggingFace weights."""
|
||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Callable, Iterable
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
import torch
|
import torch
|
||||||
@@ -288,7 +288,15 @@ class HYV3MTP(nn.Module):
|
|||||||
num_kv_heads = getattr(
|
num_kv_heads = getattr(
|
||||||
self.config, "num_key_value_heads", self.config.num_attention_heads
|
self.config, "num_key_value_heads", self.config.num_attention_heads
|
||||||
)
|
)
|
||||||
split_params_mapping = [
|
split_params_mapping: list[
|
||||||
|
tuple[
|
||||||
|
str,
|
||||||
|
str,
|
||||||
|
int,
|
||||||
|
list[tuple[str | int, int]],
|
||||||
|
Callable[[torch.Tensor], torch.Tensor] | None,
|
||||||
|
]
|
||||||
|
] = [
|
||||||
(".gate_up_proj", ".gate_and_up_proj", 2, [(1, 1), (0, 1)], None),
|
(".gate_up_proj", ".gate_and_up_proj", 2, [(1, 1), (0, 1)], None),
|
||||||
(
|
(
|
||||||
".qkv_proj",
|
".qkv_proj",
|
||||||
@@ -308,7 +316,7 @@ class HYV3MTP(nn.Module):
|
|||||||
num_experts=self.config.num_experts,
|
num_experts=self.config.num_experts,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
expert_params_mapping = {}
|
expert_params_mapping = []
|
||||||
|
|
||||||
params_dict = dict(self.named_parameters())
|
params_dict = dict(self.named_parameters())
|
||||||
|
|
||||||
@@ -354,9 +362,10 @@ class HYV3MTP(nn.Module):
|
|||||||
if name == "__skip__":
|
if name == "__skip__":
|
||||||
continue
|
continue
|
||||||
if "scale" in name:
|
if "scale" in name:
|
||||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||||
if name is None:
|
if remapped_name is None:
|
||||||
continue
|
continue
|
||||||
|
name = remapped_name
|
||||||
is_found = False
|
is_found = False
|
||||||
|
|
||||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ Supports:
|
|||||||
|
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import Annotated, Literal
|
from typing import Annotated, Literal, TypedDict
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -122,6 +122,11 @@ class HCXVisionV2VideoEmbeddingInputs(TensorSchema):
|
|||||||
HCXVisionV2VideoInputs = HCXVisionV2VideoPixelInputs | HCXVisionV2VideoEmbeddingInputs
|
HCXVisionV2VideoInputs = HCXVisionV2VideoPixelInputs | HCXVisionV2VideoEmbeddingInputs
|
||||||
|
|
||||||
|
|
||||||
|
class HCXVisionV2MultiModalInputs(TypedDict, total=False):
|
||||||
|
image: HCXVisionV2ImageInputs | None
|
||||||
|
video: HCXVisionV2VideoInputs | None
|
||||||
|
|
||||||
|
|
||||||
class HCXVisionV2ProcessingInfo(BaseProcessingInfo):
|
class HCXVisionV2ProcessingInfo(BaseProcessingInfo):
|
||||||
"""Processing info for HyperCLOVAX V2 (32B Think model)."""
|
"""Processing info for HyperCLOVAX V2 (32B Think model)."""
|
||||||
|
|
||||||
@@ -329,6 +334,7 @@ class HCXVisionV2MultiModalProcessor(
|
|||||||
if grid_thw_elem is not None:
|
if grid_thw_elem is not None:
|
||||||
# Access .data to get the actual tensor from MultiModalFieldElem
|
# Access .data to get the actual tensor from MultiModalFieldElem
|
||||||
grid_thw = grid_thw_elem.data
|
grid_thw = grid_thw_elem.data
|
||||||
|
assert isinstance(grid_thw, torch.Tensor)
|
||||||
# Qwen2.5-VL style calculation
|
# Qwen2.5-VL style calculation
|
||||||
h, w = grid_thw[1].item(), grid_thw[2].item()
|
h, w = grid_thw[1].item(), grid_thw[2].item()
|
||||||
num_tokens = (h * w) // (merge_size**2)
|
num_tokens = (h * w) // (merge_size**2)
|
||||||
@@ -340,6 +346,7 @@ class HCXVisionV2MultiModalProcessor(
|
|||||||
if grid_thw_elem is not None:
|
if grid_thw_elem is not None:
|
||||||
# Access .data to get the actual tensor from MultiModalFieldElem
|
# Access .data to get the actual tensor from MultiModalFieldElem
|
||||||
grid_thw = grid_thw_elem.data
|
grid_thw = grid_thw_elem.data
|
||||||
|
assert isinstance(grid_thw, torch.Tensor)
|
||||||
t, h, w = grid_thw[0].item(), grid_thw[1].item(), grid_thw[2].item()
|
t, h, w = grid_thw[0].item(), grid_thw[1].item(), grid_thw[2].item()
|
||||||
num_tokens = (t * h * w) // (merge_size**2)
|
num_tokens = (t * h * w) // (merge_size**2)
|
||||||
else:
|
else:
|
||||||
@@ -606,8 +613,10 @@ class HCXVisionV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
|||||||
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
|
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
|
||||||
return video_embeds.split(sizes)
|
return video_embeds.split(sizes)
|
||||||
|
|
||||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
def _parse_and_validate_multimodal_inputs(
|
||||||
modalities = {}
|
self, **kwargs: object
|
||||||
|
) -> HCXVisionV2MultiModalInputs:
|
||||||
|
modalities: HCXVisionV2MultiModalInputs = {}
|
||||||
|
|
||||||
for input_key in kwargs:
|
for input_key in kwargs:
|
||||||
if (
|
if (
|
||||||
|
|||||||
@@ -625,6 +625,7 @@ class Idefics3ForConditionalGeneration(
|
|||||||
if image_input["type"] == "image_embeds":
|
if image_input["type"] == "image_embeds":
|
||||||
return image_input["data"]
|
return image_input["data"]
|
||||||
|
|
||||||
|
assert isinstance(image_input, Idefics3ImagePixelInputs)
|
||||||
image_features = self._process_image_pixels(image_input)
|
image_features = self._process_image_pixels(image_input)
|
||||||
image_features = self.model.connector(image_features)
|
image_features = self.model.connector(image_features)
|
||||||
|
|
||||||
@@ -788,7 +789,7 @@ class Idefics3ForConditionalGeneration(
|
|||||||
path: str = "default",
|
path: str = "default",
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
image_input = self._parse_and_validate_image_input(**mm_kwargs)
|
image_input = self._parse_and_validate_image_input(**mm_kwargs)
|
||||||
assert image_input is not None and image_input["type"] == "pixel_values"
|
assert isinstance(image_input, Idefics3ImagePixelInputs)
|
||||||
image_features = self._process_image_pixels(image_input)
|
image_features = self._process_image_pixels(image_input)
|
||||||
image_features = self.model.connector(image_features)
|
image_features = self.model.connector(image_features)
|
||||||
return image_features.flatten(0, 1)
|
return image_features.flatten(0, 1)
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ from typing_extensions import Self, TypeIs
|
|||||||
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
|
from vllm.model_executor.models.interfaces_base import VllmModel
|
||||||
from vllm.utils.collection_utils import common_prefix
|
from vllm.utils.collection_utils import common_prefix
|
||||||
from vllm.utils.func_utils import supports_kw
|
from vllm.utils.func_utils import supports_kw
|
||||||
|
|
||||||
@@ -46,8 +47,9 @@ if TYPE_CHECKING:
|
|||||||
from vllm.inputs import PromptType, TokensPrompt
|
from vllm.inputs import PromptType, TokensPrompt
|
||||||
from vllm.lora.model_manager import LoRAModelManager
|
from vllm.lora.model_manager import LoRAModelManager
|
||||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||||
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||||
from vllm.model_executor.models.interfaces_base import VllmModel
|
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||||
from vllm.model_executor.models.utils import WeightsMapper
|
from vllm.model_executor.models.utils import WeightsMapper
|
||||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
|
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
|
||||||
from vllm.multimodal.registry import _ProcessorFactories
|
from vllm.multimodal.registry import _ProcessorFactories
|
||||||
@@ -128,14 +130,24 @@ class SupportsMultiModalEmbeddings(Protocol):
|
|||||||
|
|
||||||
supports_multimodal_embeddings: ClassVar[Literal[True]] = True
|
supports_multimodal_embeddings: ClassVar[Literal[True]] = True
|
||||||
|
|
||||||
|
@overload
|
||||||
def embed_input_ids(
|
def embed_input_ids(
|
||||||
self,
|
self,
|
||||||
input_ids: Tensor,
|
input_ids: Tensor,
|
||||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
multimodal_embeddings: None = None,
|
||||||
*,
|
*,
|
||||||
is_multimodal: Tensor | None = None,
|
is_multimodal: Tensor | None = None,
|
||||||
) -> Tensor: ...
|
) -> Tensor: ...
|
||||||
|
|
||||||
|
@overload
|
||||||
|
def embed_input_ids(
|
||||||
|
self,
|
||||||
|
input_ids: Tensor,
|
||||||
|
multimodal_embeddings: MultiModalEmbeddings,
|
||||||
|
*,
|
||||||
|
is_multimodal: Tensor,
|
||||||
|
) -> Tensor: ...
|
||||||
|
|
||||||
|
|
||||||
@runtime_checkable
|
@runtime_checkable
|
||||||
class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||||
@@ -243,26 +255,30 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
|||||||
torch.nn.Module: The core language model component.
|
torch.nn.Module: The core language model component.
|
||||||
"""
|
"""
|
||||||
# Cached
|
# Cached
|
||||||
if self in _language_model_by_module:
|
assert isinstance(self, nn.Module)
|
||||||
return _language_model_by_module[self]
|
self_module = self
|
||||||
|
if self_module in _language_model_by_module:
|
||||||
|
return _language_model_by_module[self_module]
|
||||||
|
|
||||||
if self._language_model_names:
|
if self._language_model_names:
|
||||||
mod = self
|
mod = self_module
|
||||||
for attr in common_prefix(
|
for attr in common_prefix(
|
||||||
[name.split(".") for name in self._language_model_names]
|
[name.split(".") for name in self._language_model_names]
|
||||||
):
|
):
|
||||||
if attr:
|
if attr:
|
||||||
mod = getattr(mod, attr)
|
mod = getattr(mod, attr)
|
||||||
|
|
||||||
if mod is not self and hasattr(mod, "embed_input_ids"):
|
candidate: object = mod
|
||||||
_language_model_by_module[self] = mod
|
if candidate is not self_module and isinstance(candidate, VllmModel):
|
||||||
return mod
|
_language_model_by_module[self_module] = candidate
|
||||||
|
return candidate
|
||||||
|
|
||||||
# Fallback
|
# Fallback
|
||||||
for mod in self.children():
|
for mod in self_module.children():
|
||||||
if hasattr(mod, "embed_input_ids"):
|
candidate = mod
|
||||||
_language_model_by_module[self] = mod
|
if isinstance(candidate, VllmModel):
|
||||||
return mod
|
_language_model_by_module[self_module] = candidate
|
||||||
|
return candidate
|
||||||
|
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
f"No language model found in {type(self).__name__}! "
|
f"No language model found in {type(self).__name__}! "
|
||||||
@@ -289,7 +305,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
|||||||
"""
|
"""
|
||||||
from .utils import StageMissingLayer, collect_children, no_init_weights
|
from .utils import StageMissingLayer, collect_children, no_init_weights
|
||||||
|
|
||||||
mm_config = vllm_config.model_config.multimodal_config
|
mm_config = vllm_config.model_config.get_multimodal_config()
|
||||||
|
|
||||||
with collect_children(self, targets=targets) as children_names: # noqa: SIM117
|
with collect_children(self, targets=targets) as children_names: # noqa: SIM117
|
||||||
with (
|
with (
|
||||||
@@ -339,7 +355,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
|||||||
else:
|
else:
|
||||||
stage_name = "_".join([*modalities, "tower"])
|
stage_name = "_".join([*modalities, "tower"])
|
||||||
|
|
||||||
mm_config = vllm_config.model_config.multimodal_config
|
mm_config = vllm_config.model_config.get_multimodal_config()
|
||||||
|
|
||||||
with collect_children(self, targets=targets) as children_names: # noqa: SIM117
|
with collect_children(self, targets=targets) as children_names: # noqa: SIM117
|
||||||
with (
|
with (
|
||||||
@@ -361,7 +377,8 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
|||||||
offloader = get_offloader()
|
offloader = get_offloader()
|
||||||
if offloader.supports_tower_offload:
|
if offloader.supports_tower_offload:
|
||||||
for name in children_names:
|
for name in children_names:
|
||||||
offloader.wrap_modules(iter([attrgetter(name)(self)]), prefix=name)
|
modules = (module for module in [attrgetter(name)(self)])
|
||||||
|
offloader.wrap_modules(modules, prefix=name)
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _mark_composite_model(
|
def _mark_composite_model(
|
||||||
@@ -434,18 +451,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
|||||||
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
|
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
|
||||||
)
|
)
|
||||||
|
|
||||||
@overload
|
|
||||||
def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...
|
|
||||||
|
|
||||||
@overload
|
|
||||||
def embed_input_ids(
|
|
||||||
self,
|
|
||||||
input_ids: Tensor,
|
|
||||||
multimodal_embeddings: MultiModalEmbeddings,
|
|
||||||
*,
|
|
||||||
is_multimodal: torch.Tensor,
|
|
||||||
) -> Tensor: ...
|
|
||||||
|
|
||||||
def _embed_text_input_ids(
|
def _embed_text_input_ids(
|
||||||
self,
|
self,
|
||||||
input_ids: Tensor,
|
input_ids: Tensor,
|
||||||
@@ -465,6 +470,24 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
|||||||
|
|
||||||
return embed_input_ids(input_ids)
|
return embed_input_ids(input_ids)
|
||||||
|
|
||||||
|
@overload
|
||||||
|
def embed_input_ids(
|
||||||
|
self,
|
||||||
|
input_ids: Tensor,
|
||||||
|
multimodal_embeddings: None = None,
|
||||||
|
*,
|
||||||
|
is_multimodal: Tensor | None = None,
|
||||||
|
) -> Tensor: ...
|
||||||
|
|
||||||
|
@overload
|
||||||
|
def embed_input_ids(
|
||||||
|
self,
|
||||||
|
input_ids: Tensor,
|
||||||
|
multimodal_embeddings: MultiModalEmbeddings,
|
||||||
|
*,
|
||||||
|
is_multimodal: Tensor,
|
||||||
|
) -> Tensor: ...
|
||||||
|
|
||||||
def embed_input_ids(
|
def embed_input_ids(
|
||||||
self,
|
self,
|
||||||
input_ids: Tensor,
|
input_ids: Tensor,
|
||||||
@@ -671,7 +694,7 @@ class SupportsLoRA(Protocol):
|
|||||||
# The `embedding_module` and `embedding_padding_modules`
|
# The `embedding_module` and `embedding_padding_modules`
|
||||||
# are empty by default.
|
# are empty by default.
|
||||||
embedding_modules: ClassVar[dict[str, str]] = {}
|
embedding_modules: ClassVar[dict[str, str]] = {}
|
||||||
packed_modules_mapping: ClassVar[dict[str, list[str]]] = {}
|
packed_modules_mapping: dict[str, list[str]] = {}
|
||||||
# Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers)
|
# Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers)
|
||||||
lora_skip_prefixes: ClassVar[list[str]] = []
|
lora_skip_prefixes: ClassVar[list[str]] = []
|
||||||
lora_manager: "LoRAModelManager | None"
|
lora_manager: "LoRAModelManager | None"
|
||||||
@@ -864,11 +887,11 @@ class HasInnerState(Protocol):
|
|||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def has_inner_state(model: object) -> TypeIs[HasInnerState]: ...
|
def has_inner_state(model: type[object]) -> TypeIs[type[HasInnerState]]: ...
|
||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def has_inner_state(model: type[object]) -> TypeIs[type[HasInnerState]]: ...
|
def has_inner_state(model: object) -> TypeIs[HasInnerState]: ...
|
||||||
|
|
||||||
|
|
||||||
def has_inner_state(
|
def has_inner_state(
|
||||||
@@ -891,11 +914,11 @@ class IsAttentionFree(Protocol):
|
|||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def is_attention_free(model: object) -> TypeIs[IsAttentionFree]: ...
|
def is_attention_free(model: type[object]) -> TypeIs[type[IsAttentionFree]]: ...
|
||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def is_attention_free(model: type[object]) -> TypeIs[type[IsAttentionFree]]: ...
|
def is_attention_free(model: object) -> TypeIs[IsAttentionFree]: ...
|
||||||
|
|
||||||
|
|
||||||
def is_attention_free(
|
def is_attention_free(
|
||||||
@@ -946,11 +969,11 @@ class IsHybrid(Protocol):
|
|||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def is_hybrid(model: object) -> TypeIs[IsHybrid]: ...
|
def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ...
|
||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ...
|
def is_hybrid(model: object) -> TypeIs[IsHybrid]: ...
|
||||||
|
|
||||||
|
|
||||||
def is_hybrid(
|
def is_hybrid(
|
||||||
@@ -1025,7 +1048,7 @@ class MixtureOfExperts(Protocol):
|
|||||||
self.expert_weights = []
|
self.expert_weights = []
|
||||||
for layer_idx, layer in enumerate(self.moe_layers):
|
for layer_idx, layer in enumerate(self.moe_layers):
|
||||||
# Register the expert weights.
|
# Register the expert weights.
|
||||||
self.expert_weights.append(layer.get_expert_weights())
|
self.expert_weights.append(list(layer.get_expert_weights()))
|
||||||
layer.set_eplb_state(
|
layer.set_eplb_state(
|
||||||
moe_layer_idx=layer_idx,
|
moe_layer_idx=layer_idx,
|
||||||
expert_load_view=expert_load_view,
|
expert_load_view=expert_load_view,
|
||||||
@@ -1083,11 +1106,11 @@ class HasNoOps(Protocol):
|
|||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def has_noops(model: object) -> TypeIs[HasNoOps]: ...
|
def has_noops(model: type[object]) -> TypeIs[type[HasNoOps]]: ...
|
||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def has_noops(model: type[object]) -> TypeIs[type[HasNoOps]]: ...
|
def has_noops(model: object) -> TypeIs[HasNoOps]: ...
|
||||||
|
|
||||||
|
|
||||||
def has_noops(
|
def has_noops(
|
||||||
@@ -1108,14 +1131,14 @@ class SupportsMambaPrefixCaching(Protocol):
|
|||||||
|
|
||||||
@overload
|
@overload
|
||||||
def supports_mamba_prefix_caching(
|
def supports_mamba_prefix_caching(
|
||||||
model: object,
|
model: type[object],
|
||||||
) -> TypeIs[SupportsMambaPrefixCaching]: ...
|
) -> TypeIs[type[SupportsMambaPrefixCaching]]: ...
|
||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def supports_mamba_prefix_caching(
|
def supports_mamba_prefix_caching(
|
||||||
model: type[object],
|
model: object,
|
||||||
) -> TypeIs[type[SupportsMambaPrefixCaching]]: ...
|
) -> TypeIs[SupportsMambaPrefixCaching]: ...
|
||||||
|
|
||||||
|
|
||||||
def supports_mamba_prefix_caching(
|
def supports_mamba_prefix_caching(
|
||||||
@@ -1136,11 +1159,11 @@ class SupportsReplaySSM(Protocol):
|
|||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def supports_replayssm(model: object) -> TypeIs[SupportsReplaySSM]: ...
|
def supports_replayssm(model: type[object]) -> TypeIs[type[SupportsReplaySSM]]: ...
|
||||||
|
|
||||||
|
|
||||||
@overload
|
@overload
|
||||||
def supports_replayssm(model: type[object]) -> TypeIs[type[SupportsReplaySSM]]: ...
|
def supports_replayssm(model: object) -> TypeIs[SupportsReplaySSM]: ...
|
||||||
|
|
||||||
|
|
||||||
def supports_replayssm(
|
def supports_replayssm(
|
||||||
@@ -1172,7 +1195,7 @@ class SupportsQuant:
|
|||||||
"""The interface required for all models that support quantization."""
|
"""The interface required for all models that support quantization."""
|
||||||
|
|
||||||
hf_to_vllm_mapper: "WeightsMapper | None" = None
|
hf_to_vllm_mapper: "WeightsMapper | None" = None
|
||||||
packed_modules_mapping: ClassVar[dict[str, list[str]]]
|
packed_modules_mapping: dict[str, list[str]]
|
||||||
quant_config: QuantizationConfig | None = None
|
quant_config: QuantizationConfig | None = None
|
||||||
|
|
||||||
def __new__(cls, *args, **kwargs) -> Self:
|
def __new__(cls, *args, **kwargs) -> Self:
|
||||||
@@ -1504,6 +1527,9 @@ class LocalArgmaxMixin:
|
|||||||
``self.draft_id_to_target_id`` (optional): nn.Parameter
|
``self.draft_id_to_target_id`` (optional): nn.Parameter
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
logits_processor: "LogitsProcessor"
|
||||||
|
lm_head: "ParallelLMHead"
|
||||||
|
|
||||||
def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||||
"""Vocab-parallel argmax with optional D2T remapping."""
|
"""Vocab-parallel argmax with optional D2T remapping."""
|
||||||
top = self.logits_processor.get_top_tokens(
|
top = self.logits_processor.get_top_tokens(
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ class VllmModel(Protocol[T_co]):
|
|||||||
|
|
||||||
|
|
||||||
def _check_vllm_model_init(model: type[object] | object) -> bool:
|
def _check_vllm_model_init(model: type[object] | object) -> bool:
|
||||||
model_init = model.__init__
|
model_init = getattr(model, "__init__") # noqa: B009
|
||||||
return supports_kw(model_init, "vllm_config")
|
return supports_kw(model_init, "vllm_config")
|
||||||
|
|
||||||
|
|
||||||
@@ -274,10 +274,11 @@ def get_attn_type(model: type[object] | object) -> AttnTypeStr:
|
|||||||
|
|
||||||
|
|
||||||
def get_score_type(model: type[object] | object) -> ScoreType:
|
def get_score_type(model: type[object] | object) -> ScoreType:
|
||||||
score_types = set()
|
score_types: set[ScoreType] = set()
|
||||||
for m in model.__mro__:
|
model_cls = model if isinstance(model, type) else type(model)
|
||||||
score_type = getattr(m, "score_type", "bi-encoder")
|
for m in model_cls.__mro__:
|
||||||
|
score_type: ScoreType = getattr(m, "score_type", "bi-encoder")
|
||||||
if score_type != "bi-encoder":
|
if score_type != "bi-encoder":
|
||||||
score_types.add(score_type)
|
score_types.add(score_type)
|
||||||
assert len(score_types) < 2
|
assert len(score_types) < 2
|
||||||
return "bi-encoder" if not score_types else list(score_types)[0]
|
return "bi-encoder" if not score_types else next(iter(score_types))
|
||||||
|
|||||||
@@ -323,7 +323,7 @@ class InternLM2Model(nn.Module):
|
|||||||
|
|
||||||
class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsQuant):
|
class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsQuant):
|
||||||
hf_to_vllm_mapper = InternLM2Model.hf_to_vllm_mapper
|
hf_to_vllm_mapper = InternLM2Model.hf_to_vllm_mapper
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping: dict[str, list[str]] = {
|
||||||
"wqkv": ["wqkv"],
|
"wqkv": ["wqkv"],
|
||||||
"gate_up_proj": ["w1", "w3"],
|
"gate_up_proj": ["w1", "w3"],
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
# Licensed under The MIT License [see LICENSE for details]
|
# Licensed under The MIT License [see LICENSE for details]
|
||||||
# --------------------------------------------------------
|
# --------------------------------------------------------
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from typing import Annotated, Literal, TypeAlias
|
from typing import Annotated, Literal, TypeAlias, TypedDict
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
import torch
|
import torch
|
||||||
@@ -22,7 +22,7 @@ from transformers.models.internvl.video_processing_internvl import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||||
from vllm.inputs import MultiModalDataDict
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.models.interns1_vit import InternS1VisionModel
|
from vllm.model_executor.models.interns1_vit import InternS1VisionModel
|
||||||
@@ -148,6 +148,11 @@ class InternS1VideoEmbeddingInputs(TensorSchema):
|
|||||||
InternS1VideoInputs: TypeAlias = InternS1VideoPixelInputs | InternS1VideoEmbeddingInputs
|
InternS1VideoInputs: TypeAlias = InternS1VideoPixelInputs | InternS1VideoEmbeddingInputs
|
||||||
|
|
||||||
|
|
||||||
|
class InternS1MultiModalInputs(TypedDict, total=False):
|
||||||
|
images: InternS1ImageInputs | None
|
||||||
|
videos: InternS1VideoInputs | None
|
||||||
|
|
||||||
|
|
||||||
def resolve_interns1_min_max_num(
|
def resolve_interns1_min_max_num(
|
||||||
min_dynamic_patch: int,
|
min_dynamic_patch: int,
|
||||||
max_dynamic_patch: int,
|
max_dynamic_patch: int,
|
||||||
@@ -312,6 +317,7 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
|
|||||||
|
|
||||||
image_overrides = mm_options.get("image")
|
image_overrides = mm_options.get("image")
|
||||||
video_overrides = mm_options.get("video")
|
video_overrides = mm_options.get("video")
|
||||||
|
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"image": self._get_dummy_images(
|
"image": self._get_dummy_images(
|
||||||
@@ -474,22 +480,24 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
|||||||
video_token = hf_processor.video_token
|
video_token = hf_processor.video_token
|
||||||
|
|
||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
video_token_id = cached_encode(tokenizer, video_token, add_special_tokens=False)
|
video_token_ids = cached_encode(
|
||||||
assert len(video_token_id) == 1
|
tokenizer, video_token, add_special_tokens=False
|
||||||
video_token_id = video_token_id[0]
|
)
|
||||||
|
assert len(video_token_ids) == 1
|
||||||
|
video_token_id = video_token_ids[0]
|
||||||
|
|
||||||
out_mm_data = out_mm_kwargs.get_data()
|
out_mm_data = out_mm_kwargs.get_data()
|
||||||
if "video_num_patches" in out_mm_data:
|
if "video_num_patches" in out_mm_data:
|
||||||
video_num_patches = out_mm_data["video_num_patches"]
|
video_num_patches_tensor = out_mm_data["video_num_patches"]
|
||||||
assert isinstance(video_num_patches, torch.Tensor)
|
assert isinstance(video_num_patches_tensor, torch.Tensor)
|
||||||
video_num_patches = video_num_patches.tolist()
|
video_num_patches: list[int] = video_num_patches_tensor.tolist()
|
||||||
else:
|
else:
|
||||||
video_num_patches = []
|
video_num_patches = []
|
||||||
|
|
||||||
if "image_num_patches" in out_mm_data:
|
if "image_num_patches" in out_mm_data:
|
||||||
image_num_patches = out_mm_data["image_num_patches"]
|
image_num_patches_tensor = out_mm_data["image_num_patches"]
|
||||||
assert isinstance(image_num_patches, torch.Tensor)
|
assert isinstance(image_num_patches_tensor, torch.Tensor)
|
||||||
image_num_patches = image_num_patches.tolist()
|
image_num_patches: list[int] = image_num_patches_tensor.tolist()
|
||||||
else:
|
else:
|
||||||
image_num_patches = []
|
image_num_patches = []
|
||||||
|
|
||||||
@@ -602,8 +610,8 @@ class InternS1ForConditionalGeneration(
|
|||||||
prefix=maybe_prefix(prefix, "language_model"),
|
prefix=maybe_prefix(prefix, "language_model"),
|
||||||
)
|
)
|
||||||
|
|
||||||
self.img_context_token_id = None
|
self.img_context_token_id: int | None = None
|
||||||
self.video_context_token_id = None
|
self.video_context_token_id: int | None = None
|
||||||
|
|
||||||
self.visual_token_mask = None
|
self.visual_token_mask = None
|
||||||
self.make_empty_intermediate_tensors = (
|
self.make_empty_intermediate_tensors = (
|
||||||
@@ -756,8 +764,10 @@ class InternS1ForConditionalGeneration(
|
|||||||
]
|
]
|
||||||
return image_embeds.split(image_feature_sizes)
|
return image_embeds.split(image_feature_sizes)
|
||||||
|
|
||||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
def _parse_and_validate_multimodal_inputs(
|
||||||
modalities = {}
|
self, **kwargs: object
|
||||||
|
) -> InternS1MultiModalInputs:
|
||||||
|
modalities: InternS1MultiModalInputs = {}
|
||||||
|
|
||||||
# Preserve the order of modalities if there are multiple of them
|
# Preserve the order of modalities if there are multiple of them
|
||||||
# from the order of kwargs.
|
# from the order of kwargs.
|
||||||
@@ -789,12 +799,14 @@ class InternS1ForConditionalGeneration(
|
|||||||
for modality in modalities:
|
for modality in modalities:
|
||||||
if modality == "images":
|
if modality == "images":
|
||||||
image_input = modalities["images"]
|
image_input = modalities["images"]
|
||||||
image_embeddings = self._process_vision_input(image_input)
|
if image_input is not None:
|
||||||
multimodal_embeddings += tuple(image_embeddings)
|
image_embeddings = self._process_vision_input(image_input)
|
||||||
|
multimodal_embeddings += tuple(image_embeddings)
|
||||||
if modality == "videos":
|
if modality == "videos":
|
||||||
video_input = modalities["videos"]
|
video_input = modalities["videos"]
|
||||||
video_embeddings = self._process_vision_input(video_input)
|
if video_input is not None:
|
||||||
multimodal_embeddings += tuple(video_embeddings)
|
video_embeddings = self._process_vision_input(video_input)
|
||||||
|
multimodal_embeddings += tuple(video_embeddings)
|
||||||
|
|
||||||
return multimodal_embeddings
|
return multimodal_embeddings
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@
|
|||||||
|
|
||||||
import functools
|
import functools
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from typing import Any
|
from typing import Any, ClassVar
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
@@ -321,6 +321,14 @@ class InternS1ProMoeAttention(nn.Module):
|
|||||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
attn_kwargs: dict[str, Any] = (
|
||||||
|
{
|
||||||
|
"layer_idx": extract_layer_index(prefix),
|
||||||
|
"dual_chunk_attention_config": dual_chunk_attention_config,
|
||||||
|
}
|
||||||
|
if dual_chunk_attention_config
|
||||||
|
else {}
|
||||||
|
)
|
||||||
self.attn = Attention(
|
self.attn = Attention(
|
||||||
self.num_heads,
|
self.num_heads,
|
||||||
self.head_dim,
|
self.head_dim,
|
||||||
@@ -329,12 +337,7 @@ class InternS1ProMoeAttention(nn.Module):
|
|||||||
cache_config=cache_config,
|
cache_config=cache_config,
|
||||||
quant_config=quant_config,
|
quant_config=quant_config,
|
||||||
prefix=f"{prefix}.attn",
|
prefix=f"{prefix}.attn",
|
||||||
**{
|
**attn_kwargs,
|
||||||
"layer_idx": extract_layer_index(prefix),
|
|
||||||
"dual_chunk_attention_config": dual_chunk_attention_config,
|
|
||||||
}
|
|
||||||
if dual_chunk_attention_config
|
|
||||||
else {},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||||
@@ -489,6 +492,9 @@ class InternS1ProMoeLLMForCausalLM(Qwen3MoeForCausalLM):
|
|||||||
|
|
||||||
|
|
||||||
class InternS1ProMoeMixtureOfExperts(MixtureOfExperts):
|
class InternS1ProMoeMixtureOfExperts(MixtureOfExperts):
|
||||||
|
language_model: InternS1ProMoeLLMForCausalLM
|
||||||
|
num_local_physical_experts: int
|
||||||
|
|
||||||
def update_physical_experts_metadata(
|
def update_physical_experts_metadata(
|
||||||
self,
|
self,
|
||||||
num_physical_experts: int,
|
num_physical_experts: int,
|
||||||
@@ -538,8 +544,8 @@ class InternS1ProMoeMixtureOfExperts(MixtureOfExperts):
|
|||||||
class InternS1ProForConditionalGeneration(
|
class InternS1ProForConditionalGeneration(
|
||||||
Qwen3VLForConditionalGeneration, InternS1ProMoeMixtureOfExperts
|
Qwen3VLForConditionalGeneration, InternS1ProMoeMixtureOfExperts
|
||||||
):
|
):
|
||||||
is_3d_moe_weight: bool = True
|
is_3d_moe_weight: ClassVar[bool] = True
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping: dict[str, list[str]] = {
|
||||||
"qkv_proj": [
|
"qkv_proj": [
|
||||||
"q_proj",
|
"q_proj",
|
||||||
"k_proj",
|
"k_proj",
|
||||||
@@ -559,7 +565,7 @@ class InternS1ProForConditionalGeneration(
|
|||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||||
super(Qwen3VLForConditionalGeneration, self).__init__()
|
super(Qwen3VLForConditionalGeneration, self).__init__()
|
||||||
config: PretrainedConfig = vllm_config.model_config.hf_config
|
config: PretrainedConfig = vllm_config.model_config.hf_config
|
||||||
multimodal_config = vllm_config.model_config.multimodal_config
|
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
self.multimodal_config = multimodal_config
|
self.multimodal_config = multimodal_config
|
||||||
@@ -601,8 +607,8 @@ class InternS1ProForConditionalGeneration(
|
|||||||
# Set MoE hyperparameters
|
# Set MoE hyperparameters
|
||||||
self.set_moe_parameters()
|
self.set_moe_parameters()
|
||||||
|
|
||||||
def get_frope_params_map(self) -> str:
|
def get_frope_params_map(self) -> dict[str, str]:
|
||||||
mapper = {}
|
mapper: dict[str, str] = {}
|
||||||
for name, params in self.language_model.model.named_parameters():
|
for name, params in self.language_model.model.named_parameters():
|
||||||
if "rotary_emb.sin_coef" in name:
|
if "rotary_emb.sin_coef" in name:
|
||||||
mapper["language_model.model.rotary_emb.sin_coef"] = (
|
mapper["language_model.model.rotary_emb.sin_coef"] = (
|
||||||
|
|||||||
@@ -476,7 +476,7 @@ class InternS2MobiusForConditionalGeneration(InternS2PreviewForConditionalGenera
|
|||||||
if getattr(config, "image_token_index", None) is None:
|
if getattr(config, "image_token_index", None) is None:
|
||||||
config.image_token_index = config.image_token_id
|
config.image_token_index = config.image_token_id
|
||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
multimodal_config = vllm_config.model_config.multimodal_config
|
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
self.model_config = vllm_config.model_config
|
self.model_config = vllm_config.model_config
|
||||||
|
|||||||
@@ -10,14 +10,14 @@
|
|||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from functools import cached_property
|
from functools import cached_property
|
||||||
from typing import Annotated, Any, Literal, TypeAlias, TypeVar
|
from typing import Annotated, Any, Literal, TypeAlias, TypedDict, TypeVar
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
from transformers import BatchFeature, PretrainedConfig
|
from transformers import BatchFeature, PretrainedConfig
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.multimodal import BaseDummyOptions
|
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||||
from vllm.inputs import MultiModalDataDict
|
from vllm.inputs import MultiModalDataDict
|
||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||||
@@ -128,6 +128,11 @@ class InternVLVideoEmbeddingInputs(TensorSchema):
|
|||||||
InternVLVideoInputs: TypeAlias = InternVLVideoPixelInputs | InternVLVideoEmbeddingInputs
|
InternVLVideoInputs: TypeAlias = InternVLVideoPixelInputs | InternVLVideoEmbeddingInputs
|
||||||
|
|
||||||
|
|
||||||
|
class InternVLMultiModalInputs(TypedDict, total=False):
|
||||||
|
images: InternVLImageInputs | None
|
||||||
|
videos: InternVLVideoInputs | None
|
||||||
|
|
||||||
|
|
||||||
class BaseInternVLProcessingInfo(BaseProcessingInfo):
|
class BaseInternVLProcessingInfo(BaseProcessingInfo):
|
||||||
"""Basic image-only ProcessingInfo for InternVL-style models."""
|
"""Basic image-only ProcessingInfo for InternVL-style models."""
|
||||||
|
|
||||||
@@ -286,6 +291,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
|||||||
if isinstance(images, ImageEmbeddingItems):
|
if isinstance(images, ImageEmbeddingItems):
|
||||||
feature_size = images.get_feature_size(item_idx)
|
feature_size = images.get_feature_size(item_idx)
|
||||||
else:
|
else:
|
||||||
|
assert isinstance(images, ImageProcessorItems)
|
||||||
image_size = images.get_image_size(item_idx)
|
image_size = images.get_image_size(item_idx)
|
||||||
feature_size = self.info.get_num_image_tokens(
|
feature_size = self.info.get_num_image_tokens(
|
||||||
image_width=image_size.width,
|
image_width=image_size.width,
|
||||||
@@ -433,6 +439,9 @@ class InternVLDummyInputsBuilder(
|
|||||||
)
|
)
|
||||||
num_videos = mm_counts.get("video", 0)
|
num_videos = mm_counts.get("video", 0)
|
||||||
video_overrides = mm_options.get("video")
|
video_overrides = mm_options.get("video")
|
||||||
|
assert video_overrides is None or isinstance(
|
||||||
|
video_overrides, VideoDummyOptions
|
||||||
|
)
|
||||||
dummy_video = {
|
dummy_video = {
|
||||||
"video": self._get_dummy_videos(
|
"video": self._get_dummy_videos(
|
||||||
width=image_size,
|
width=image_size,
|
||||||
@@ -504,9 +513,9 @@ class InternVLMultiModalProcessor(
|
|||||||
tokenizer = self.info.get_tokenizer()
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
|
||||||
if "video_num_patches" in out_mm_data:
|
if "video_num_patches" in out_mm_data:
|
||||||
video_num_patches = out_mm_data["video_num_patches"]
|
video_num_patches_tensor = out_mm_data["video_num_patches"]
|
||||||
assert isinstance(video_num_patches, torch.Tensor)
|
assert isinstance(video_num_patches_tensor, torch.Tensor)
|
||||||
video_num_patches = video_num_patches.tolist()
|
video_num_patches: list[int] = video_num_patches_tensor.tolist()
|
||||||
else:
|
else:
|
||||||
video_num_patches = []
|
video_num_patches = []
|
||||||
|
|
||||||
@@ -591,7 +600,7 @@ class InternVLChatModel(
|
|||||||
|
|
||||||
config = vllm_config.model_config.hf_config
|
config = vllm_config.model_config.hf_config
|
||||||
quant_config = vllm_config.quant_config
|
quant_config = vllm_config.quant_config
|
||||||
multimodal_config = vllm_config.model_config.multimodal_config
|
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
self.multimodal_config = multimodal_config
|
self.multimodal_config = multimodal_config
|
||||||
@@ -621,15 +630,15 @@ class InternVLChatModel(
|
|||||||
prefix=maybe_prefix(prefix, "language_model"),
|
prefix=maybe_prefix(prefix, "language_model"),
|
||||||
)
|
)
|
||||||
|
|
||||||
self.img_context_token_id = None
|
self.img_context_token_id: int | None = None
|
||||||
self.video_context_token_id = None
|
self.video_context_token_id: int | None = None
|
||||||
|
|
||||||
self.make_empty_intermediate_tensors = (
|
self.make_empty_intermediate_tensors = (
|
||||||
self.language_model.make_empty_intermediate_tensors
|
self.language_model.make_empty_intermediate_tensors
|
||||||
)
|
)
|
||||||
|
|
||||||
def _patch_quant_config(
|
def _patch_quant_config(
|
||||||
self, config: PretrainedConfig, quant_config: QuantizationConfig
|
self, config: PretrainedConfig, quant_config: QuantizationConfig | None
|
||||||
):
|
):
|
||||||
# the awq models from OpenGVLab missing `modules_to_not_convert`
|
# the awq models from OpenGVLab missing `modules_to_not_convert`
|
||||||
# patch the quant_config to add `modules_to_not_convert` back
|
# patch the quant_config to add `modules_to_not_convert` back
|
||||||
@@ -743,7 +752,7 @@ class InternVLChatModel(
|
|||||||
|
|
||||||
def _parse_and_validate_video_input(
|
def _parse_and_validate_video_input(
|
||||||
self, **kwargs: object
|
self, **kwargs: object
|
||||||
) -> InternVLVideoPixelInputs | None:
|
) -> InternVLVideoInputs | None:
|
||||||
pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)
|
pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)
|
||||||
video_num_patches = kwargs.pop("video_num_patches", None)
|
video_num_patches = kwargs.pop("video_num_patches", None)
|
||||||
video_embeds = kwargs.pop("image_embeds", None)
|
video_embeds = kwargs.pop("image_embeds", None)
|
||||||
@@ -804,8 +813,10 @@ class InternVLChatModel(
|
|||||||
]
|
]
|
||||||
return image_embeds.split(image_feature_sizes)
|
return image_embeds.split(image_feature_sizes)
|
||||||
|
|
||||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
def _parse_and_validate_multimodal_inputs(
|
||||||
modalities = {}
|
self, **kwargs: object
|
||||||
|
) -> InternVLMultiModalInputs:
|
||||||
|
modalities: InternVLMultiModalInputs = {}
|
||||||
|
|
||||||
# Preserve the order of modalities if there are multiple of them
|
# Preserve the order of modalities if there are multiple of them
|
||||||
# from the order of kwargs.
|
# from the order of kwargs.
|
||||||
@@ -834,12 +845,14 @@ class InternVLChatModel(
|
|||||||
for modality in modalities:
|
for modality in modalities:
|
||||||
if modality == "images":
|
if modality == "images":
|
||||||
image_input = modalities["images"]
|
image_input = modalities["images"]
|
||||||
image_embeddings = self._process_vision_input(image_input)
|
if image_input is not None:
|
||||||
multimodal_embeddings += tuple(image_embeddings)
|
image_embeddings = self._process_vision_input(image_input)
|
||||||
|
multimodal_embeddings += tuple(image_embeddings)
|
||||||
if modality == "videos":
|
if modality == "videos":
|
||||||
video_input = modalities["videos"]
|
video_input = modalities["videos"]
|
||||||
video_embeddings = self._process_vision_input(video_input)
|
if video_input is not None:
|
||||||
multimodal_embeddings += tuple(video_embeddings)
|
video_embeddings = self._process_vision_input(video_input)
|
||||||
|
multimodal_embeddings += tuple(video_embeddings)
|
||||||
|
|
||||||
return multimodal_embeddings
|
return multimodal_embeddings
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Callable, Iterable
|
||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -487,11 +487,14 @@ class IQuestLoopCoderModel(nn.Module):
|
|||||||
continue
|
continue
|
||||||
if name.endswith("scale"):
|
if name.endswith("scale"):
|
||||||
# Remapping the name of FP8 kv-scale.
|
# Remapping the name of FP8 kv-scale.
|
||||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||||
if name is None:
|
if remapped_name is None:
|
||||||
continue
|
continue
|
||||||
|
name = remapped_name
|
||||||
param = params_dict[name]
|
param = params_dict[name]
|
||||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
weight_loader: Callable[..., None] = getattr(
|
||||||
|
param, "weight_loader", default_weight_loader
|
||||||
|
)
|
||||||
if weight_loader == default_weight_loader:
|
if weight_loader == default_weight_loader:
|
||||||
weight_loader(param, loaded_weight)
|
weight_loader(param, loaded_weight)
|
||||||
else:
|
else:
|
||||||
@@ -518,9 +521,10 @@ class IQuestLoopCoderModel(nn.Module):
|
|||||||
if name.endswith(".bias") and name not in params_dict:
|
if name.endswith(".bias") and name not in params_dict:
|
||||||
continue
|
continue
|
||||||
# Remapping the name of FP8 kv-scale.
|
# Remapping the name of FP8 kv-scale.
|
||||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||||
if name is None:
|
if remapped_name is None:
|
||||||
continue
|
continue
|
||||||
|
name = remapped_name
|
||||||
param = params_dict[name]
|
param = params_dict[name]
|
||||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||||
weight_loader(param, loaded_weight)
|
weight_loader(param, loaded_weight)
|
||||||
|
|||||||
@@ -901,7 +901,11 @@ class IsaacForConditionalGeneration(
|
|||||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||||
offset = mm_feature.mm_position.offset
|
offset = mm_feature.mm_position.offset
|
||||||
if mm_feature.modality == "image":
|
if mm_feature.modality == "image":
|
||||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
mm_data = mm_feature.data
|
||||||
|
assert mm_data is not None
|
||||||
|
grid_thw = mm_data["image_grid_thw"].data
|
||||||
|
assert isinstance(grid_thw, torch.Tensor)
|
||||||
|
t, h, w = grid_thw.tolist()
|
||||||
assert t == 1, f"Image must have 1 frame, got {t}"
|
assert t == 1, f"Image must have 1 frame, got {t}"
|
||||||
yield offset, h // spatial_merge_size, w // spatial_merge_size
|
yield offset, h // spatial_merge_size, w // spatial_merge_size
|
||||||
else:
|
else:
|
||||||
@@ -912,7 +916,7 @@ class IsaacForConditionalGeneration(
|
|||||||
input_tokens: list[int],
|
input_tokens: list[int],
|
||||||
mm_features: list[MultiModalFeatureSpec],
|
mm_features: list[MultiModalFeatureSpec],
|
||||||
) -> tuple[torch.Tensor, int]:
|
) -> tuple[torch.Tensor, int]:
|
||||||
llm_pos_ids_list = []
|
llm_pos_ids_list: list[np.ndarray] = []
|
||||||
st = 0
|
st = 0
|
||||||
for offset, llm_grid_h, llm_grid_w in self.iter_mm_grid_hw(
|
for offset, llm_grid_h, llm_grid_w in self.iter_mm_grid_hw(
|
||||||
input_tokens, mm_features
|
input_tokens, mm_features
|
||||||
|
|||||||
@@ -1753,7 +1753,7 @@ class Qwen3VLForConditionalGeneration(
|
|||||||
SupportsEagle3,
|
SupportsEagle3,
|
||||||
SupportsMultiModalPruning,
|
SupportsMultiModalPruning,
|
||||||
):
|
):
|
||||||
packed_modules_mapping = {
|
packed_modules_mapping: dict[str, list[str]] = {
|
||||||
"qkv_proj": [
|
"qkv_proj": [
|
||||||
"q_proj",
|
"q_proj",
|
||||||
"k_proj",
|
"k_proj",
|
||||||
|
|||||||
Reference in New Issue
Block a user