[Mypy] Fix mypy typing for model interfaces and H/I models (#54079)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-08-28 16:49:19 -04:00
committed by GitHub
co-authored by Wentao Ye
parent 74850f9f66
commit 21fa2c5a2a
23 changed files with 222 additions and 137 deletions
-2
View File
@@ -112,8 +112,6 @@ SEPARATE_GROUPS = [
] ]
EXCLUDE = [ EXCLUDE = [
r"vllm/model_executor/models/[hH]",
r"vllm/model_executor/models/[iI]",
r"vllm/model_executor/models/[jJ]", r"vllm/model_executor/models/[jJ]",
r"vllm/model_executor/models/[kK]", r"vllm/model_executor/models/[kK]",
r"vllm/model_executor/models/[lL]", r"vllm/model_executor/models/[lL]",
+1 -1
View File
@@ -430,7 +430,7 @@ class ChatGLMBaseModel(nn.Module):
class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP, SupportsQuant): class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP, SupportsQuant):
packed_modules_mapping = { packed_modules_mapping: dict[str, list[str]] = {
"query_key_value": ["query_key_value"], "query_key_value": ["query_key_value"],
"dense_h_to_4h": ["dense_h_to_4h"], "dense_h_to_4h": ["dense_h_to_4h"],
} }
+1 -1
View File
@@ -417,7 +417,7 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
orig_to_new_substr={"_quantizer.": None}, orig_to_new_substr={"_quantizer.": None},
orig_to_new_prefix={"lm_head": None}, orig_to_new_prefix={"lm_head": None},
) )
packed_modules_mapping = { packed_modules_mapping: dict[str, list[str]] = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"], "qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"], "gate_up_proj": ["gate_proj", "up_proj"],
} }
+1 -1
View File
@@ -339,7 +339,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
".up_proj": (".gate_up_proj", 1), ".up_proj": (".gate_up_proj", 1),
} }
) )
packed_modules_mapping = { packed_modules_mapping: dict[str, list[str]] = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"], "qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"], "gate_up_proj": ["gate_proj", "up_proj"],
} }
+1 -1
View File
@@ -1019,7 +1019,7 @@ class Gemma4ForConditionalGeneration(
mm_prefix_clamp_sliding_window: bool = True mm_prefix_clamp_sliding_window: bool = True
supports_tower_connector_lora = True supports_tower_connector_lora = True
packed_modules_mapping = { packed_modules_mapping: dict[str, list[str]] = {
"qkv_proj": [ "qkv_proj": [
"q_proj", "q_proj",
"k_proj", "k_proj",
+1 -1
View File
@@ -395,7 +395,7 @@ class GraniteModel(nn.Module):
class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
hf_to_vllm_mapper = GraniteModel.hf_to_vllm_mapper hf_to_vllm_mapper = GraniteModel.hf_to_vllm_mapper
# LoRA specific attributes # LoRA specific attributes
packed_modules_mapping = { packed_modules_mapping: dict[str, list[str]] = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"], "qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"], "gate_up_proj": ["gate_proj", "up_proj"],
} }
@@ -587,7 +587,7 @@ class GraniteMoeHybridForCausalLM(
SupportsQuant, SupportsQuant,
SupportsMambaPrefixCaching, SupportsMambaPrefixCaching,
): ):
packed_modules_mapping = { packed_modules_mapping: dict[str, list[str]] = {
"qkv_proj": [ "qkv_proj": [
"q_proj", "q_proj",
"k_proj", "k_proj",
+1
View File
@@ -114,6 +114,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
if isinstance(images, ImageEmbeddingItems): if isinstance(images, ImageEmbeddingItems):
feature_size = images.get_feature_size(item_idx) feature_size = images.get_feature_size(item_idx)
else: else:
assert isinstance(images, ImageProcessorItems)
image_size = images.get_image_size(item_idx) image_size = images.get_image_size(item_idx)
feature_size = self.info.get_num_image_tokens( feature_size = self.info.get_num_image_tokens(
image_width=image_size.width, image_width=image_size.width,
+2 -2
View File
@@ -248,7 +248,7 @@ class HrmTextDecoderLayer(nn.Module):
self, self,
config: PretrainedConfig, config: PretrainedConfig,
layer_idx_in_stack: int, layer_idx_in_stack: int,
stack_kind: str, stack_kind: Literal["L", "H"],
cache_config: CacheConfig | None = None, cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None, quant_config: QuantizationConfig | None = None,
prefix: str = "", prefix: str = "",
@@ -312,7 +312,7 @@ class HrmTextStack(nn.Module):
def __init__( def __init__(
self, self,
config: PretrainedConfig, config: PretrainedConfig,
stack_kind: str, stack_kind: Literal["L", "H"],
cache_config: CacheConfig | None = None, cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None, quant_config: QuantizationConfig | None = None,
prefix: str = "", prefix: str = "",
+12 -11
View File
@@ -161,6 +161,7 @@ class HYV3MoEFused(nn.Module):
prefix=f"{prefix}.gate", prefix=f"{prefix}.gate",
) )
self.shared_mlp: HYV3FeedForward | None
if config.num_shared_experts > 0: if config.num_shared_experts > 0:
self.shared_mlp = HYV3FeedForward( self.shared_mlp = HYV3FeedForward(
hidden_size=config.hidden_size, hidden_size=config.hidden_size,
@@ -467,28 +468,27 @@ class HYV3Model(nn.Module, MixtureOfExperts):
# Set MoE hyperparameters # Set MoE hyperparameters
self.num_expert_groups = 1 self.num_expert_groups = 1
self.moe_layers = [] self.moe_layers = []
example_layer = None example_layer: HYV3MoEFused | None = None
for layer in self.layers: for layer in self.layers:
if isinstance(layer, PPMissingLayer): if isinstance(layer, PPMissingLayer):
continue continue
assert isinstance(layer, HYV3DecoderLayer) assert isinstance(layer, HYV3DecoderLayer)
if layer.block_type == "moe": if layer.block_type == "moe":
assert isinstance(layer.mlp, HYV3MoEFused)
example_layer = layer.mlp example_layer = layer.mlp
self.moe_layers.append(layer.mlp.experts) self.moe_layers.append(example_layer.experts)
if example_layer is None: if example_layer is None:
self.num_moe_layers = 0 self.num_moe_layers = 0
raise RuntimeError("No MoE layer found in model.layers.") raise RuntimeError("No MoE layer found in model.layers.")
self.num_moe_layers = len(self.moe_layers) self.num_moe_layers = len(self.moe_layers)
self.num_logical_experts = getattr(example_layer, "n_logical_experts", None) self.num_logical_experts = example_layer.n_logical_experts
self.num_physical_experts = getattr(example_layer, "n_physical_experts", None) self.num_physical_experts = example_layer.n_physical_experts
self.num_local_physical_experts = getattr( self.num_local_physical_experts = example_layer.n_local_physical_experts
example_layer, "n_local_physical_experts", None self.num_routed_experts = example_layer.n_routed_experts
) self.num_redundant_experts = example_layer.n_redundant_experts
self.num_routed_experts = getattr(example_layer, "n_routed_experts", None)
self.num_redundant_experts = getattr(example_layer, "n_redundant_experts", None)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids) return self.embed_tokens(input_ids)
@@ -572,9 +572,10 @@ class HYV3Model(nn.Module, MixtureOfExperts):
continue continue
if "scale" in name: if "scale" in name:
# Remapping the name of FP8 kv-scale. # Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict) remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if name is None: if remapped_name is None:
continue continue
name = remapped_name
is_found = False is_found = False
for param_name, weight_name, shard_id in stacked_params_mapping: for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name: if weight_name not in name:
+14 -5
View File
@@ -24,7 +24,7 @@
# limitations under the License. # limitations under the License.
"""Inference-only HY V3 MTP model compatible with HuggingFace weights.""" """Inference-only HY V3 MTP model compatible with HuggingFace weights."""
from collections.abc import Iterable from collections.abc import Callable, Iterable
import regex as re import regex as re
import torch import torch
@@ -288,7 +288,15 @@ class HYV3MTP(nn.Module):
num_kv_heads = getattr( num_kv_heads = getattr(
self.config, "num_key_value_heads", self.config.num_attention_heads self.config, "num_key_value_heads", self.config.num_attention_heads
) )
split_params_mapping = [ split_params_mapping: list[
tuple[
str,
str,
int,
list[tuple[str | int, int]],
Callable[[torch.Tensor], torch.Tensor] | None,
]
] = [
(".gate_up_proj", ".gate_and_up_proj", 2, [(1, 1), (0, 1)], None), (".gate_up_proj", ".gate_and_up_proj", 2, [(1, 1), (0, 1)], None),
( (
".qkv_proj", ".qkv_proj",
@@ -308,7 +316,7 @@ class HYV3MTP(nn.Module):
num_experts=self.config.num_experts, num_experts=self.config.num_experts,
) )
else: else:
expert_params_mapping = {} expert_params_mapping = []
params_dict = dict(self.named_parameters()) params_dict = dict(self.named_parameters())
@@ -354,9 +362,10 @@ class HYV3MTP(nn.Module):
if name == "__skip__": if name == "__skip__":
continue continue
if "scale" in name: if "scale" in name:
name = maybe_remap_kv_scale_name(name, params_dict) remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if name is None: if remapped_name is None:
continue continue
name = remapped_name
is_found = False is_found = False
for param_name, weight_name, shard_id in stacked_params_mapping: for param_name, weight_name, shard_id in stacked_params_mapping:
@@ -12,7 +12,7 @@ Supports:
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from functools import partial from functools import partial
from typing import Annotated, Literal from typing import Annotated, Literal, TypedDict
import torch import torch
import torch.nn as nn import torch.nn as nn
@@ -122,6 +122,11 @@ class HCXVisionV2VideoEmbeddingInputs(TensorSchema):
HCXVisionV2VideoInputs = HCXVisionV2VideoPixelInputs | HCXVisionV2VideoEmbeddingInputs HCXVisionV2VideoInputs = HCXVisionV2VideoPixelInputs | HCXVisionV2VideoEmbeddingInputs
class HCXVisionV2MultiModalInputs(TypedDict, total=False):
image: HCXVisionV2ImageInputs | None
video: HCXVisionV2VideoInputs | None
class HCXVisionV2ProcessingInfo(BaseProcessingInfo): class HCXVisionV2ProcessingInfo(BaseProcessingInfo):
"""Processing info for HyperCLOVAX V2 (32B Think model).""" """Processing info for HyperCLOVAX V2 (32B Think model)."""
@@ -329,6 +334,7 @@ class HCXVisionV2MultiModalProcessor(
if grid_thw_elem is not None: if grid_thw_elem is not None:
# Access .data to get the actual tensor from MultiModalFieldElem # Access .data to get the actual tensor from MultiModalFieldElem
grid_thw = grid_thw_elem.data grid_thw = grid_thw_elem.data
assert isinstance(grid_thw, torch.Tensor)
# Qwen2.5-VL style calculation # Qwen2.5-VL style calculation
h, w = grid_thw[1].item(), grid_thw[2].item() h, w = grid_thw[1].item(), grid_thw[2].item()
num_tokens = (h * w) // (merge_size**2) num_tokens = (h * w) // (merge_size**2)
@@ -340,6 +346,7 @@ class HCXVisionV2MultiModalProcessor(
if grid_thw_elem is not None: if grid_thw_elem is not None:
# Access .data to get the actual tensor from MultiModalFieldElem # Access .data to get the actual tensor from MultiModalFieldElem
grid_thw = grid_thw_elem.data grid_thw = grid_thw_elem.data
assert isinstance(grid_thw, torch.Tensor)
t, h, w = grid_thw[0].item(), grid_thw[1].item(), grid_thw[2].item() t, h, w = grid_thw[0].item(), grid_thw[1].item(), grid_thw[2].item()
num_tokens = (t * h * w) // (merge_size**2) num_tokens = (t * h * w) // (merge_size**2)
else: else:
@@ -606,8 +613,10 @@ class HCXVisionV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist() sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
return video_embeds.split(sizes) return video_embeds.split(sizes)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: def _parse_and_validate_multimodal_inputs(
modalities = {} self, **kwargs: object
) -> HCXVisionV2MultiModalInputs:
modalities: HCXVisionV2MultiModalInputs = {}
for input_key in kwargs: for input_key in kwargs:
if ( if (
+2 -1
View File
@@ -625,6 +625,7 @@ class Idefics3ForConditionalGeneration(
if image_input["type"] == "image_embeds": if image_input["type"] == "image_embeds":
return image_input["data"] return image_input["data"]
assert isinstance(image_input, Idefics3ImagePixelInputs)
image_features = self._process_image_pixels(image_input) image_features = self._process_image_pixels(image_input)
image_features = self.model.connector(image_features) image_features = self.model.connector(image_features)
@@ -788,7 +789,7 @@ class Idefics3ForConditionalGeneration(
path: str = "default", path: str = "default",
) -> torch.Tensor: ) -> torch.Tensor:
image_input = self._parse_and_validate_image_input(**mm_kwargs) image_input = self._parse_and_validate_image_input(**mm_kwargs)
assert image_input is not None and image_input["type"] == "pixel_values" assert isinstance(image_input, Idefics3ImagePixelInputs)
image_features = self._process_image_pixels(image_input) image_features = self._process_image_pixels(image_input)
image_features = self.model.connector(image_features) image_features = self.model.connector(image_features)
return image_features.flatten(0, 1) return image_features.flatten(0, 1)
+70 -44
View File
@@ -32,6 +32,7 @@ from typing_extensions import Self, TypeIs
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.utils.collection_utils import common_prefix from vllm.utils.collection_utils import common_prefix
from vllm.utils.func_utils import supports_kw from vllm.utils.func_utils import supports_kw
@@ -46,8 +47,9 @@ if TYPE_CHECKING:
from vllm.inputs import PromptType, TokensPrompt from vllm.inputs import PromptType, TokensPrompt
from vllm.lora.model_manager import LoRAModelManager from vllm.lora.model_manager import LoRAModelManager
from vllm.model_executor.layers.fused_moe import MoERunner from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.model_executor.models.interfaces_base import VllmModel from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.utils import WeightsMapper from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
from vllm.multimodal.registry import _ProcessorFactories from vllm.multimodal.registry import _ProcessorFactories
@@ -128,14 +130,24 @@ class SupportsMultiModalEmbeddings(Protocol):
supports_multimodal_embeddings: ClassVar[Literal[True]] = True supports_multimodal_embeddings: ClassVar[Literal[True]] = True
@overload
def embed_input_ids( def embed_input_ids(
self, self,
input_ids: Tensor, input_ids: Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None, multimodal_embeddings: None = None,
*, *,
is_multimodal: Tensor | None = None, is_multimodal: Tensor | None = None,
) -> Tensor: ... ) -> Tensor: ...
@overload
def embed_input_ids(
self,
input_ids: Tensor,
multimodal_embeddings: MultiModalEmbeddings,
*,
is_multimodal: Tensor,
) -> Tensor: ...
@runtime_checkable @runtime_checkable
class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol): class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
@@ -243,26 +255,30 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
torch.nn.Module: The core language model component. torch.nn.Module: The core language model component.
""" """
# Cached # Cached
if self in _language_model_by_module: assert isinstance(self, nn.Module)
return _language_model_by_module[self] self_module = self
if self_module in _language_model_by_module:
return _language_model_by_module[self_module]
if self._language_model_names: if self._language_model_names:
mod = self mod = self_module
for attr in common_prefix( for attr in common_prefix(
[name.split(".") for name in self._language_model_names] [name.split(".") for name in self._language_model_names]
): ):
if attr: if attr:
mod = getattr(mod, attr) mod = getattr(mod, attr)
if mod is not self and hasattr(mod, "embed_input_ids"): candidate: object = mod
_language_model_by_module[self] = mod if candidate is not self_module and isinstance(candidate, VllmModel):
return mod _language_model_by_module[self_module] = candidate
return candidate
# Fallback # Fallback
for mod in self.children(): for mod in self_module.children():
if hasattr(mod, "embed_input_ids"): candidate = mod
_language_model_by_module[self] = mod if isinstance(candidate, VllmModel):
return mod _language_model_by_module[self_module] = candidate
return candidate
raise NotImplementedError( raise NotImplementedError(
f"No language model found in {type(self).__name__}! " f"No language model found in {type(self).__name__}! "
@@ -289,7 +305,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
""" """
from .utils import StageMissingLayer, collect_children, no_init_weights from .utils import StageMissingLayer, collect_children, no_init_weights
mm_config = vllm_config.model_config.multimodal_config mm_config = vllm_config.model_config.get_multimodal_config()
with collect_children(self, targets=targets) as children_names: # noqa: SIM117 with collect_children(self, targets=targets) as children_names: # noqa: SIM117
with ( with (
@@ -339,7 +355,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
else: else:
stage_name = "_".join([*modalities, "tower"]) stage_name = "_".join([*modalities, "tower"])
mm_config = vllm_config.model_config.multimodal_config mm_config = vllm_config.model_config.get_multimodal_config()
with collect_children(self, targets=targets) as children_names: # noqa: SIM117 with collect_children(self, targets=targets) as children_names: # noqa: SIM117
with ( with (
@@ -361,7 +377,8 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
offloader = get_offloader() offloader = get_offloader()
if offloader.supports_tower_offload: if offloader.supports_tower_offload:
for name in children_names: for name in children_names:
offloader.wrap_modules(iter([attrgetter(name)(self)]), prefix=name) modules = (module for module in [attrgetter(name)(self)])
offloader.wrap_modules(modules, prefix=name)
@contextmanager @contextmanager
def _mark_composite_model( def _mark_composite_model(
@@ -434,18 +451,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
num_connector_tokens if isinstance(num_connector_tokens, int) else None, num_connector_tokens if isinstance(num_connector_tokens, int) else None,
) )
@overload
def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...
@overload
def embed_input_ids(
self,
input_ids: Tensor,
multimodal_embeddings: MultiModalEmbeddings,
*,
is_multimodal: torch.Tensor,
) -> Tensor: ...
def _embed_text_input_ids( def _embed_text_input_ids(
self, self,
input_ids: Tensor, input_ids: Tensor,
@@ -465,6 +470,24 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
return embed_input_ids(input_ids) return embed_input_ids(input_ids)
@overload
def embed_input_ids(
self,
input_ids: Tensor,
multimodal_embeddings: None = None,
*,
is_multimodal: Tensor | None = None,
) -> Tensor: ...
@overload
def embed_input_ids(
self,
input_ids: Tensor,
multimodal_embeddings: MultiModalEmbeddings,
*,
is_multimodal: Tensor,
) -> Tensor: ...
def embed_input_ids( def embed_input_ids(
self, self,
input_ids: Tensor, input_ids: Tensor,
@@ -671,7 +694,7 @@ class SupportsLoRA(Protocol):
# The `embedding_module` and `embedding_padding_modules` # The `embedding_module` and `embedding_padding_modules`
# are empty by default. # are empty by default.
embedding_modules: ClassVar[dict[str, str]] = {} embedding_modules: ClassVar[dict[str, str]] = {}
packed_modules_mapping: ClassVar[dict[str, list[str]]] = {} packed_modules_mapping: dict[str, list[str]] = {}
# Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers) # Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers)
lora_skip_prefixes: ClassVar[list[str]] = [] lora_skip_prefixes: ClassVar[list[str]] = []
lora_manager: "LoRAModelManager | None" lora_manager: "LoRAModelManager | None"
@@ -864,11 +887,11 @@ class HasInnerState(Protocol):
@overload @overload
def has_inner_state(model: object) -> TypeIs[HasInnerState]: ... def has_inner_state(model: type[object]) -> TypeIs[type[HasInnerState]]: ...
@overload @overload
def has_inner_state(model: type[object]) -> TypeIs[type[HasInnerState]]: ... def has_inner_state(model: object) -> TypeIs[HasInnerState]: ...
def has_inner_state( def has_inner_state(
@@ -891,11 +914,11 @@ class IsAttentionFree(Protocol):
@overload @overload
def is_attention_free(model: object) -> TypeIs[IsAttentionFree]: ... def is_attention_free(model: type[object]) -> TypeIs[type[IsAttentionFree]]: ...
@overload @overload
def is_attention_free(model: type[object]) -> TypeIs[type[IsAttentionFree]]: ... def is_attention_free(model: object) -> TypeIs[IsAttentionFree]: ...
def is_attention_free( def is_attention_free(
@@ -946,11 +969,11 @@ class IsHybrid(Protocol):
@overload @overload
def is_hybrid(model: object) -> TypeIs[IsHybrid]: ... def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ...
@overload @overload
def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ... def is_hybrid(model: object) -> TypeIs[IsHybrid]: ...
def is_hybrid( def is_hybrid(
@@ -1025,7 +1048,7 @@ class MixtureOfExperts(Protocol):
self.expert_weights = [] self.expert_weights = []
for layer_idx, layer in enumerate(self.moe_layers): for layer_idx, layer in enumerate(self.moe_layers):
# Register the expert weights. # Register the expert weights.
self.expert_weights.append(layer.get_expert_weights()) self.expert_weights.append(list(layer.get_expert_weights()))
layer.set_eplb_state( layer.set_eplb_state(
moe_layer_idx=layer_idx, moe_layer_idx=layer_idx,
expert_load_view=expert_load_view, expert_load_view=expert_load_view,
@@ -1083,11 +1106,11 @@ class HasNoOps(Protocol):
@overload @overload
def has_noops(model: object) -> TypeIs[HasNoOps]: ... def has_noops(model: type[object]) -> TypeIs[type[HasNoOps]]: ...
@overload @overload
def has_noops(model: type[object]) -> TypeIs[type[HasNoOps]]: ... def has_noops(model: object) -> TypeIs[HasNoOps]: ...
def has_noops( def has_noops(
@@ -1108,14 +1131,14 @@ class SupportsMambaPrefixCaching(Protocol):
@overload @overload
def supports_mamba_prefix_caching( def supports_mamba_prefix_caching(
model: object, model: type[object],
) -> TypeIs[SupportsMambaPrefixCaching]: ... ) -> TypeIs[type[SupportsMambaPrefixCaching]]: ...
@overload @overload
def supports_mamba_prefix_caching( def supports_mamba_prefix_caching(
model: type[object], model: object,
) -> TypeIs[type[SupportsMambaPrefixCaching]]: ... ) -> TypeIs[SupportsMambaPrefixCaching]: ...
def supports_mamba_prefix_caching( def supports_mamba_prefix_caching(
@@ -1136,11 +1159,11 @@ class SupportsReplaySSM(Protocol):
@overload @overload
def supports_replayssm(model: object) -> TypeIs[SupportsReplaySSM]: ... def supports_replayssm(model: type[object]) -> TypeIs[type[SupportsReplaySSM]]: ...
@overload @overload
def supports_replayssm(model: type[object]) -> TypeIs[type[SupportsReplaySSM]]: ... def supports_replayssm(model: object) -> TypeIs[SupportsReplaySSM]: ...
def supports_replayssm( def supports_replayssm(
@@ -1172,7 +1195,7 @@ class SupportsQuant:
"""The interface required for all models that support quantization.""" """The interface required for all models that support quantization."""
hf_to_vllm_mapper: "WeightsMapper | None" = None hf_to_vllm_mapper: "WeightsMapper | None" = None
packed_modules_mapping: ClassVar[dict[str, list[str]]] packed_modules_mapping: dict[str, list[str]]
quant_config: QuantizationConfig | None = None quant_config: QuantizationConfig | None = None
def __new__(cls, *args, **kwargs) -> Self: def __new__(cls, *args, **kwargs) -> Self:
@@ -1504,6 +1527,9 @@ class LocalArgmaxMixin:
``self.draft_id_to_target_id`` (optional): nn.Parameter ``self.draft_id_to_target_id`` (optional): nn.Parameter
""" """
logits_processor: "LogitsProcessor"
lm_head: "ParallelLMHead"
def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor: def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Vocab-parallel argmax with optional D2T remapping.""" """Vocab-parallel argmax with optional D2T remapping."""
top = self.logits_processor.get_top_tokens( top = self.logits_processor.get_top_tokens(
@@ -57,7 +57,7 @@ class VllmModel(Protocol[T_co]):
def _check_vllm_model_init(model: type[object] | object) -> bool: def _check_vllm_model_init(model: type[object] | object) -> bool:
model_init = model.__init__ model_init = getattr(model, "__init__") # noqa: B009
return supports_kw(model_init, "vllm_config") return supports_kw(model_init, "vllm_config")
@@ -274,10 +274,11 @@ def get_attn_type(model: type[object] | object) -> AttnTypeStr:
def get_score_type(model: type[object] | object) -> ScoreType: def get_score_type(model: type[object] | object) -> ScoreType:
score_types = set() score_types: set[ScoreType] = set()
for m in model.__mro__: model_cls = model if isinstance(model, type) else type(model)
score_type = getattr(m, "score_type", "bi-encoder") for m in model_cls.__mro__:
score_type: ScoreType = getattr(m, "score_type", "bi-encoder")
if score_type != "bi-encoder": if score_type != "bi-encoder":
score_types.add(score_type) score_types.add(score_type)
assert len(score_types) < 2 assert len(score_types) < 2
return "bi-encoder" if not score_types else list(score_types)[0] return "bi-encoder" if not score_types else next(iter(score_types))
+1 -1
View File
@@ -323,7 +323,7 @@ class InternLM2Model(nn.Module):
class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsQuant): class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsQuant):
hf_to_vllm_mapper = InternLM2Model.hf_to_vllm_mapper hf_to_vllm_mapper = InternLM2Model.hf_to_vllm_mapper
packed_modules_mapping = { packed_modules_mapping: dict[str, list[str]] = {
"wqkv": ["wqkv"], "wqkv": ["wqkv"],
"gate_up_proj": ["w1", "w3"], "gate_up_proj": ["w1", "w3"],
} }
+31 -19
View File
@@ -7,7 +7,7 @@
# Licensed under The MIT License [see LICENSE for details] # Licensed under The MIT License [see LICENSE for details]
# -------------------------------------------------------- # --------------------------------------------------------
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, TypeAlias from typing import Annotated, Literal, TypeAlias, TypedDict
import regex as re import regex as re
import torch import torch
@@ -22,7 +22,7 @@ from transformers.models.internvl.video_processing_internvl import (
) )
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.inputs import MultiModalDataDict from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.interns1_vit import InternS1VisionModel from vllm.model_executor.models.interns1_vit import InternS1VisionModel
@@ -148,6 +148,11 @@ class InternS1VideoEmbeddingInputs(TensorSchema):
InternS1VideoInputs: TypeAlias = InternS1VideoPixelInputs | InternS1VideoEmbeddingInputs InternS1VideoInputs: TypeAlias = InternS1VideoPixelInputs | InternS1VideoEmbeddingInputs
class InternS1MultiModalInputs(TypedDict, total=False):
images: InternS1ImageInputs | None
videos: InternS1VideoInputs | None
def resolve_interns1_min_max_num( def resolve_interns1_min_max_num(
min_dynamic_patch: int, min_dynamic_patch: int,
max_dynamic_patch: int, max_dynamic_patch: int,
@@ -312,6 +317,7 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
image_overrides = mm_options.get("image") image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video") video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return { return {
"image": self._get_dummy_images( "image": self._get_dummy_images(
@@ -474,22 +480,24 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
video_token = hf_processor.video_token video_token = hf_processor.video_token
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
video_token_id = cached_encode(tokenizer, video_token, add_special_tokens=False) video_token_ids = cached_encode(
assert len(video_token_id) == 1 tokenizer, video_token, add_special_tokens=False
video_token_id = video_token_id[0] )
assert len(video_token_ids) == 1
video_token_id = video_token_ids[0]
out_mm_data = out_mm_kwargs.get_data() out_mm_data = out_mm_kwargs.get_data()
if "video_num_patches" in out_mm_data: if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"] video_num_patches_tensor = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches, torch.Tensor) assert isinstance(video_num_patches_tensor, torch.Tensor)
video_num_patches = video_num_patches.tolist() video_num_patches: list[int] = video_num_patches_tensor.tolist()
else: else:
video_num_patches = [] video_num_patches = []
if "image_num_patches" in out_mm_data: if "image_num_patches" in out_mm_data:
image_num_patches = out_mm_data["image_num_patches"] image_num_patches_tensor = out_mm_data["image_num_patches"]
assert isinstance(image_num_patches, torch.Tensor) assert isinstance(image_num_patches_tensor, torch.Tensor)
image_num_patches = image_num_patches.tolist() image_num_patches: list[int] = image_num_patches_tensor.tolist()
else: else:
image_num_patches = [] image_num_patches = []
@@ -602,8 +610,8 @@ class InternS1ForConditionalGeneration(
prefix=maybe_prefix(prefix, "language_model"), prefix=maybe_prefix(prefix, "language_model"),
) )
self.img_context_token_id = None self.img_context_token_id: int | None = None
self.video_context_token_id = None self.video_context_token_id: int | None = None
self.visual_token_mask = None self.visual_token_mask = None
self.make_empty_intermediate_tensors = ( self.make_empty_intermediate_tensors = (
@@ -756,8 +764,10 @@ class InternS1ForConditionalGeneration(
] ]
return image_embeds.split(image_feature_sizes) return image_embeds.split(image_feature_sizes)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: def _parse_and_validate_multimodal_inputs(
modalities = {} self, **kwargs: object
) -> InternS1MultiModalInputs:
modalities: InternS1MultiModalInputs = {}
# Preserve the order of modalities if there are multiple of them # Preserve the order of modalities if there are multiple of them
# from the order of kwargs. # from the order of kwargs.
@@ -789,12 +799,14 @@ class InternS1ForConditionalGeneration(
for modality in modalities: for modality in modalities:
if modality == "images": if modality == "images":
image_input = modalities["images"] image_input = modalities["images"]
image_embeddings = self._process_vision_input(image_input) if image_input is not None:
multimodal_embeddings += tuple(image_embeddings) image_embeddings = self._process_vision_input(image_input)
multimodal_embeddings += tuple(image_embeddings)
if modality == "videos": if modality == "videos":
video_input = modalities["videos"] video_input = modalities["videos"]
video_embeddings = self._process_vision_input(video_input) if video_input is not None:
multimodal_embeddings += tuple(video_embeddings) video_embeddings = self._process_vision_input(video_input)
multimodal_embeddings += tuple(video_embeddings)
return multimodal_embeddings return multimodal_embeddings
+18 -12
View File
@@ -26,7 +26,7 @@
import functools import functools
from collections.abc import Iterable from collections.abc import Iterable
from typing import Any from typing import Any, ClassVar
import torch import torch
from torch import nn from torch import nn
@@ -321,6 +321,14 @@ class InternS1ProMoeAttention(nn.Module):
dual_chunk_attention_config=dual_chunk_attention_config, dual_chunk_attention_config=dual_chunk_attention_config,
) )
attn_kwargs: dict[str, Any] = (
{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {}
)
self.attn = Attention( self.attn = Attention(
self.num_heads, self.num_heads,
self.head_dim, self.head_dim,
@@ -329,12 +337,7 @@ class InternS1ProMoeAttention(nn.Module):
cache_config=cache_config, cache_config=cache_config,
quant_config=quant_config, quant_config=quant_config,
prefix=f"{prefix}.attn", prefix=f"{prefix}.attn",
**{ **attn_kwargs,
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {},
) )
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
@@ -489,6 +492,9 @@ class InternS1ProMoeLLMForCausalLM(Qwen3MoeForCausalLM):
class InternS1ProMoeMixtureOfExperts(MixtureOfExperts): class InternS1ProMoeMixtureOfExperts(MixtureOfExperts):
language_model: InternS1ProMoeLLMForCausalLM
num_local_physical_experts: int
def update_physical_experts_metadata( def update_physical_experts_metadata(
self, self,
num_physical_experts: int, num_physical_experts: int,
@@ -538,8 +544,8 @@ class InternS1ProMoeMixtureOfExperts(MixtureOfExperts):
class InternS1ProForConditionalGeneration( class InternS1ProForConditionalGeneration(
Qwen3VLForConditionalGeneration, InternS1ProMoeMixtureOfExperts Qwen3VLForConditionalGeneration, InternS1ProMoeMixtureOfExperts
): ):
is_3d_moe_weight: bool = True is_3d_moe_weight: ClassVar[bool] = True
packed_modules_mapping = { packed_modules_mapping: dict[str, list[str]] = {
"qkv_proj": [ "qkv_proj": [
"q_proj", "q_proj",
"k_proj", "k_proj",
@@ -559,7 +565,7 @@ class InternS1ProForConditionalGeneration(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super(Qwen3VLForConditionalGeneration, self).__init__() super(Qwen3VLForConditionalGeneration, self).__init__()
config: PretrainedConfig = vllm_config.model_config.hf_config config: PretrainedConfig = vllm_config.model_config.hf_config
multimodal_config = vllm_config.model_config.multimodal_config multimodal_config = vllm_config.model_config.get_multimodal_config()
self.config = config self.config = config
self.multimodal_config = multimodal_config self.multimodal_config = multimodal_config
@@ -601,8 +607,8 @@ class InternS1ProForConditionalGeneration(
# Set MoE hyperparameters # Set MoE hyperparameters
self.set_moe_parameters() self.set_moe_parameters()
def get_frope_params_map(self) -> str: def get_frope_params_map(self) -> dict[str, str]:
mapper = {} mapper: dict[str, str] = {}
for name, params in self.language_model.model.named_parameters(): for name, params in self.language_model.model.named_parameters():
if "rotary_emb.sin_coef" in name: if "rotary_emb.sin_coef" in name:
mapper["language_model.model.rotary_emb.sin_coef"] = ( mapper["language_model.model.rotary_emb.sin_coef"] = (
@@ -476,7 +476,7 @@ class InternS2MobiusForConditionalGeneration(InternS2PreviewForConditionalGenera
if getattr(config, "image_token_index", None) is None: if getattr(config, "image_token_index", None) is None:
config.image_token_index = config.image_token_id config.image_token_index = config.image_token_id
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config multimodal_config = vllm_config.model_config.get_multimodal_config()
self.config = config self.config = config
self.model_config = vllm_config.model_config self.model_config = vllm_config.model_config
+29 -16
View File
@@ -10,14 +10,14 @@
from abc import abstractmethod from abc import abstractmethod
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from functools import cached_property from functools import cached_property
from typing import Annotated, Any, Literal, TypeAlias, TypeVar from typing import Annotated, Any, Literal, TypeAlias, TypedDict, TypeVar
import torch import torch
import torch.nn as nn import torch.nn as nn
from transformers import BatchFeature, PretrainedConfig from transformers import BatchFeature, PretrainedConfig
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.inputs import MultiModalDataDict from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
@@ -128,6 +128,11 @@ class InternVLVideoEmbeddingInputs(TensorSchema):
InternVLVideoInputs: TypeAlias = InternVLVideoPixelInputs | InternVLVideoEmbeddingInputs InternVLVideoInputs: TypeAlias = InternVLVideoPixelInputs | InternVLVideoEmbeddingInputs
class InternVLMultiModalInputs(TypedDict, total=False):
images: InternVLImageInputs | None
videos: InternVLVideoInputs | None
class BaseInternVLProcessingInfo(BaseProcessingInfo): class BaseInternVLProcessingInfo(BaseProcessingInfo):
"""Basic image-only ProcessingInfo for InternVL-style models.""" """Basic image-only ProcessingInfo for InternVL-style models."""
@@ -286,6 +291,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
if isinstance(images, ImageEmbeddingItems): if isinstance(images, ImageEmbeddingItems):
feature_size = images.get_feature_size(item_idx) feature_size = images.get_feature_size(item_idx)
else: else:
assert isinstance(images, ImageProcessorItems)
image_size = images.get_image_size(item_idx) image_size = images.get_image_size(item_idx)
feature_size = self.info.get_num_image_tokens( feature_size = self.info.get_num_image_tokens(
image_width=image_size.width, image_width=image_size.width,
@@ -433,6 +439,9 @@ class InternVLDummyInputsBuilder(
) )
num_videos = mm_counts.get("video", 0) num_videos = mm_counts.get("video", 0)
video_overrides = mm_options.get("video") video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(
video_overrides, VideoDummyOptions
)
dummy_video = { dummy_video = {
"video": self._get_dummy_videos( "video": self._get_dummy_videos(
width=image_size, width=image_size,
@@ -504,9 +513,9 @@ class InternVLMultiModalProcessor(
tokenizer = self.info.get_tokenizer() tokenizer = self.info.get_tokenizer()
if "video_num_patches" in out_mm_data: if "video_num_patches" in out_mm_data:
video_num_patches = out_mm_data["video_num_patches"] video_num_patches_tensor = out_mm_data["video_num_patches"]
assert isinstance(video_num_patches, torch.Tensor) assert isinstance(video_num_patches_tensor, torch.Tensor)
video_num_patches = video_num_patches.tolist() video_num_patches: list[int] = video_num_patches_tensor.tolist()
else: else:
video_num_patches = [] video_num_patches = []
@@ -591,7 +600,7 @@ class InternVLChatModel(
config = vllm_config.model_config.hf_config config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config multimodal_config = vllm_config.model_config.get_multimodal_config()
self.config = config self.config = config
self.multimodal_config = multimodal_config self.multimodal_config = multimodal_config
@@ -621,15 +630,15 @@ class InternVLChatModel(
prefix=maybe_prefix(prefix, "language_model"), prefix=maybe_prefix(prefix, "language_model"),
) )
self.img_context_token_id = None self.img_context_token_id: int | None = None
self.video_context_token_id = None self.video_context_token_id: int | None = None
self.make_empty_intermediate_tensors = ( self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors self.language_model.make_empty_intermediate_tensors
) )
def _patch_quant_config( def _patch_quant_config(
self, config: PretrainedConfig, quant_config: QuantizationConfig self, config: PretrainedConfig, quant_config: QuantizationConfig | None
): ):
# the awq models from OpenGVLab missing `modules_to_not_convert` # the awq models from OpenGVLab missing `modules_to_not_convert`
# patch the quant_config to add `modules_to_not_convert` back # patch the quant_config to add `modules_to_not_convert` back
@@ -743,7 +752,7 @@ class InternVLChatModel(
def _parse_and_validate_video_input( def _parse_and_validate_video_input(
self, **kwargs: object self, **kwargs: object
) -> InternVLVideoPixelInputs | None: ) -> InternVLVideoInputs | None:
pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None) pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)
video_num_patches = kwargs.pop("video_num_patches", None) video_num_patches = kwargs.pop("video_num_patches", None)
video_embeds = kwargs.pop("image_embeds", None) video_embeds = kwargs.pop("image_embeds", None)
@@ -804,8 +813,10 @@ class InternVLChatModel(
] ]
return image_embeds.split(image_feature_sizes) return image_embeds.split(image_feature_sizes)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: def _parse_and_validate_multimodal_inputs(
modalities = {} self, **kwargs: object
) -> InternVLMultiModalInputs:
modalities: InternVLMultiModalInputs = {}
# Preserve the order of modalities if there are multiple of them # Preserve the order of modalities if there are multiple of them
# from the order of kwargs. # from the order of kwargs.
@@ -834,12 +845,14 @@ class InternVLChatModel(
for modality in modalities: for modality in modalities:
if modality == "images": if modality == "images":
image_input = modalities["images"] image_input = modalities["images"]
image_embeddings = self._process_vision_input(image_input) if image_input is not None:
multimodal_embeddings += tuple(image_embeddings) image_embeddings = self._process_vision_input(image_input)
multimodal_embeddings += tuple(image_embeddings)
if modality == "videos": if modality == "videos":
video_input = modalities["videos"] video_input = modalities["videos"]
video_embeddings = self._process_vision_input(video_input) if video_input is not None:
multimodal_embeddings += tuple(video_embeddings) video_embeddings = self._process_vision_input(video_input)
multimodal_embeddings += tuple(video_embeddings)
return multimodal_embeddings return multimodal_embeddings
+10 -6
View File
@@ -16,7 +16,7 @@
from __future__ import annotations from __future__ import annotations
from collections.abc import Iterable from collections.abc import Callable, Iterable
from dataclasses import replace from dataclasses import replace
from typing import Any from typing import Any
@@ -487,11 +487,14 @@ class IQuestLoopCoderModel(nn.Module):
continue continue
if name.endswith("scale"): if name.endswith("scale"):
# Remapping the name of FP8 kv-scale. # Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict) remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if name is None: if remapped_name is None:
continue continue
name = remapped_name
param = params_dict[name] param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader: Callable[..., None] = getattr(
param, "weight_loader", default_weight_loader
)
if weight_loader == default_weight_loader: if weight_loader == default_weight_loader:
weight_loader(param, loaded_weight) weight_loader(param, loaded_weight)
else: else:
@@ -518,9 +521,10 @@ class IQuestLoopCoderModel(nn.Module):
if name.endswith(".bias") and name not in params_dict: if name.endswith(".bias") and name not in params_dict:
continue continue
# Remapping the name of FP8 kv-scale. # Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict) remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if name is None: if remapped_name is None:
continue continue
name = remapped_name
param = params_dict[name] param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight) weight_loader(param, loaded_weight)
+6 -2
View File
@@ -901,7 +901,11 @@ class IsaacForConditionalGeneration(
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
offset = mm_feature.mm_position.offset offset = mm_feature.mm_position.offset
if mm_feature.modality == "image": if mm_feature.modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist() mm_data = mm_feature.data
assert mm_data is not None
grid_thw = mm_data["image_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
t, h, w = grid_thw.tolist()
assert t == 1, f"Image must have 1 frame, got {t}" assert t == 1, f"Image must have 1 frame, got {t}"
yield offset, h // spatial_merge_size, w // spatial_merge_size yield offset, h // spatial_merge_size, w // spatial_merge_size
else: else:
@@ -912,7 +916,7 @@ class IsaacForConditionalGeneration(
input_tokens: list[int], input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec], mm_features: list[MultiModalFeatureSpec],
) -> tuple[torch.Tensor, int]: ) -> tuple[torch.Tensor, int]:
llm_pos_ids_list = [] llm_pos_ids_list: list[np.ndarray] = []
st = 0 st = 0
for offset, llm_grid_h, llm_grid_w in self.iter_mm_grid_hw( for offset, llm_grid_h, llm_grid_w in self.iter_mm_grid_hw(
input_tokens, mm_features input_tokens, mm_features
+1 -1
View File
@@ -1753,7 +1753,7 @@ class Qwen3VLForConditionalGeneration(
SupportsEagle3, SupportsEagle3,
SupportsMultiModalPruning, SupportsMultiModalPruning,
): ):
packed_modules_mapping = { packed_modules_mapping: dict[str, list[str]] = {
"qkv_proj": [ "qkv_proj": [
"q_proj", "q_proj",
"k_proj", "k_proj",