[Mypy Fix] Mypy fix for "vllm/model_executor/models/[cC][dD]" (#52003)

This commit is contained in:
Wentao Ye
2026-08-13 11:33:55 +08:00
committed by GitHub
parent 15227b934a
commit 9a276d6375
24 changed files with 153 additions and 66 deletions
-3
View File
@@ -111,10 +111,7 @@ SEPARATE_GROUPS = [
"tests/v1/worker",
]
# TODO(woosuk): Include the code from Megatron and HuggingFace.
EXCLUDE = [
r"vllm/model_executor/models/[cC]",
r"vllm/model_executor/models/[dD]",
r"vllm/model_executor/models/[eE]",
r"vllm/model_executor/models/[fF]",
r"vllm/model_executor/models/[gG]",
+3
View File
@@ -142,6 +142,7 @@ class CLIPProcessingInfo(BaseProcessingInfo):
pooler_config = self.ctx.model_config.pooler_config
assert pooler_config is not None
assert pooler_config.seq_pooling_type is not None
return get_num_selected_vision_tokens(
vision_encoder_info.get_num_image_tokens(
@@ -854,6 +855,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
feature_select_strategy: VisionFeatureSelectStrategy | None = None,
) -> torch.Tensor:
if feature_select_strategy is None:
assert self.pooler_config.seq_pooling_type is not None
feature_select_strategy = _get_vision_feature_select_strategy(
self.pooler_config.seq_pooling_type
)
@@ -961,6 +963,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
raise RuntimeError("PP is not supported for this model")
# Multimodal inputs
assert inputs_embeds is not None
if not self._is_text_input:
return inputs_embeds
@@ -178,6 +178,7 @@ class Cohere2MoeAttention(nn.Module):
self.max_position_embeddings = getattr(
config, "model_max_length", None
) or getattr(config, "max_position_embeddings", 8192)
assert isinstance(self.max_position_embeddings, int)
self.qkv_proj = QKVParallelLinear(
self.hidden_size,
self.head_dim,
@@ -283,6 +284,7 @@ class Cohere2Moe(nn.Module):
prefix=f"{prefix}.gate",
)
self.shared_experts: Cohere2MoeMLP | None
if hasattr(config, "num_shared_experts") and config.num_shared_experts > 0:
self.shared_experts = Cohere2MoeMLP(
config=config,
+1 -1
View File
@@ -416,7 +416,7 @@ class Cohere2VisionForConditionalGeneration(
)
def _patch_quant_config(
self, config: PretrainedConfig, quant_config: QuantizationConfig
self, config: PretrainedConfig, quant_config: QuantizationConfig | None
):
# the awq models from OpenGVLab missing `modules_to_not_convert`
# patch the quant_config to add `modules_to_not_convert` back
+28 -13
View File
@@ -55,6 +55,7 @@ from vllm.transformers_utils.processors.cohere_asr import (
CohereASRFeatureExtractor,
CohereASRProcessor,
)
from vllm.utils.collection_utils import is_list_of
from vllm.v1.attention.backend import (
AttentionType,
)
@@ -183,6 +184,7 @@ class CohereASRAttention(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor | None = None,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
@@ -241,7 +243,7 @@ class CohereASRCrossAttention(CohereASRAttention):
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor | None,
encoder_hidden_states: torch.Tensor | None = None,
) -> torch.Tensor:
q, _ = self.q_proj(hidden_states)
@@ -837,7 +839,7 @@ class CausalConv1D(nn.Conv1d):
out_channels: int,
kernel_size: int,
stride: int = 1,
padding: str | int = 0,
padding: str | int | list[int] | None = 0,
dilation: int = 1,
groups: int = 1,
bias: bool = True,
@@ -845,6 +847,8 @@ class CausalConv1D(nn.Conv1d):
device=None,
dtype=None,
) -> None:
self._left_padding: int
self._right_padding: int
if padding is None:
self._left_padding = kernel_size - 1
self._right_padding = stride - 1
@@ -903,7 +907,7 @@ class ConformerConvolution(nn.Module):
d_model: int,
kernel_size: int,
norm_type: str = "batch_norm",
conv_context_size: int | None = None,
conv_context_size: int | list[int] | None = None,
pointwise_activation: str = "glu_",
use_bias: bool = True,
) -> None:
@@ -1158,6 +1162,7 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention):
q, k, v = self.forward_qkv(query, key, value)
q = q.transpose(1, 2) # (batch, time1, head, d_k)
assert pos_emb is not None
n_batch_pos = pos_emb.size(0)
p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k)
p = p.transpose(1, 2) # (batch, head, time1, d_k)
@@ -1213,7 +1218,7 @@ class ConformerLayer(torch.nn.Module):
n_heads: int = 4,
conv_kernel_size: int = 31,
conv_norm_type: str = "batch_norm",
conv_context_size: int | None = None,
conv_context_size: int | list[int] | None = None,
pos_bias_u: nn.Parameter | torch.Tensor | None = None,
pos_bias_v: nn.Parameter | torch.Tensor | None = None,
att_context_size: list[int] | None = None,
@@ -1382,6 +1387,7 @@ class ConformerEncoder(nn.Module):
conv_kernel_size=conv_kernel_size,
)
self.xscale: float | None
if xscaling:
self.xscale = math.sqrt(d_model)
else:
@@ -1633,9 +1639,12 @@ class ConformerEncoder(nn.Module):
) -> tuple[list[list[int]], list[int], list[float], list[int]]:
# convert att_context_size to a standard list of lists
if att_context_size:
att_context_size_all = list(att_context_size)
if isinstance(att_context_size_all[0], int):
att_context_size_all = [att_context_size_all]
if is_list_of(att_context_size, int, check="all"):
att_context_size_all = [att_context_size]
elif is_list_of(att_context_size, list, check="all"):
att_context_size_all = att_context_size
else:
raise ValueError("att_context_size cannot mix nested and flat values")
for i, att_cs in enumerate(att_context_size_all):
if att_context_style == "chunked_limited":
if att_cs[0] > 0 and att_cs[0] % (att_cs[1] + 1) > 0:
@@ -1684,6 +1693,7 @@ class ConformerEncoder(nn.Module):
if conv_context_size == "causal":
conv_context_size = [conv_kernel_size - 1, 0]
else:
assert isinstance(conv_context_size, list)
total = conv_context_size[0] + conv_context_size[1] + 1
if total != conv_kernel_size:
raise ValueError(
@@ -1940,7 +1950,7 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
):
if mm_data:
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
mm_data = dict(audio=mm_data.pop("audios"))
mm_data = dict(audio=dict(mm_data).pop("audios"))
mm_kwargs = dict(
**mm_kwargs,
sampling_rate=feature_extractor.sampling_rate,
@@ -2045,7 +2055,6 @@ class CohereAsrForConditionalGeneration(
# (which is different from "_") and encode("▁ABC") ignores the first token
# so the prompt_text is unreliable. However, prompt_token_ids can be used
# to get prompt_text but it wont have the first token "▁".
prompt_text = None
prompt_token_ids = cls._get_default_prompt_token_ids(
tokenizer,
model_config,
@@ -2053,7 +2062,6 @@ class CohereAsrForConditionalGeneration(
)
return TokensPrompt(
prompt=prompt_text,
prompt_token_ids=prompt_token_ids,
multi_modal_data={"audio": (audio, stt_config.sample_rate)},
)
@@ -2199,7 +2207,9 @@ class CohereAsrForConditionalGeneration(
audio_input, seq_lens = self._parse_and_validate_audio_input(**kwargs)
if hasattr(audio_input, "input_features"):
out = self.model.get_encoder_outputs(audio_input["input_features"])
out = self.model.get_encoder_outputs(
audio_input["input_features"], seq_lens
)
else:
out = self.model.get_encoder_outputs(audio_input, seq_lens)
@@ -2219,12 +2229,17 @@ class CohereAsrForConditionalGeneration(
f"Incorrect type of audio features. Got type: {type(input_features)}"
)
if not isinstance(length, torch.Tensor):
raise ValueError("Audio feature lengths must be a tensor.")
if isinstance(input_features, torch.Tensor):
seq_lens = length.reshape(-1)
else:
if not is_list_of(input_features, torch.Tensor, check="all"):
raise ValueError("All audio features must be tensors.")
input_features = [
feat.to(self.dtype).squeeze(0).transpose(1, 0)
for feat in input_features
feature.to(self.dtype).squeeze(0).transpose(1, 0)
for feature in input_features
]
seq_lens = length.reshape(-1)
input_features = torch.nn.utils.rnn.pad_sequence(
+3 -1
View File
@@ -58,6 +58,7 @@ class CohereEagleModel(nn.Module):
start_layer_id: int = 0,
) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
self.config = vllm_config.speculative_config.draft_model_config.hf_config
self.quant_config = get_draft_quant_config(vllm_config)
@@ -137,6 +138,7 @@ class CohereEagleModel(nn.Module):
class EagleCohereForCausalLM(CohereForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
assert vllm_config.speculative_config is not None
self.config = vllm_config.speculative_config.draft_model_config.hf_config
# Flags checked by the speculative proposer to decide whether to share
# embed_tokens / lm_head with the target model. Cohere EAGLE checkpoints
@@ -160,7 +162,7 @@ class EagleCohereForCausalLM(CohereForCausalLM):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
@@ -33,6 +33,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
)
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.configs.colmodernvbert import ColModernVBertConfig
from .interfaces import (
@@ -161,6 +162,7 @@ class ColModernVBertMultiModalProcessor(
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
text_encoding = tokenizer(
prompt,
return_tensors="pt",
+4 -1
View File
@@ -25,6 +25,7 @@ from transformers import BatchFeature, PaliGemmaProcessor
from vllm.config import VllmConfig
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -232,7 +233,9 @@ class ColPaliModel(
loaded.add(f"custom_text_proj.{param_name}")
# Update pooler projector for the lazy-creation path
self.pooler.head.projector = self.custom_text_proj
head = self.pooler.head
assert isinstance(head, TokenPoolerHead)
head.projector = self.custom_text_proj
# Mark pooler projector params as loaded
if hasattr(self, "pooler") and hasattr(self.pooler, "head"):
+1
View File
@@ -154,6 +154,7 @@ class CohereAttention(nn.Module):
self.max_position_embeddings = getattr(
config, "model_max_length", None
) or getattr(config, "max_position_embeddings", 8192)
assert isinstance(self.max_position_embeddings, int)
self.use_qk_norm = getattr(config, "use_qk_norm", False)
self.qkv_proj = QKVParallelLinear(
self.hidden_size,
+11 -2
View File
@@ -9,6 +9,7 @@ if TYPE_CHECKING:
from transformers import PretrainedConfig
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.config.cache import MambaDType
logger = init_logger(__name__)
@@ -452,6 +453,7 @@ class JambaForSequenceClassificationConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
pooler_config = model_config.pooler_config
assert pooler_config is not None
if pooler_config.use_activation is None:
pooler_config.use_activation = False
@@ -518,6 +520,7 @@ class JinaVLForSequenceClassificationConfig(VerifyAndUpdateConfig):
config = model_config.hf_config
config.num_labels = 1
pooler_config = model_config.pooler_config
assert pooler_config is not None
if pooler_config.logit_mean is None:
pooler_config.logit_mean = 2.65
@@ -540,6 +543,7 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig):
if pooling_type is None:
raise ValueError(f"pool_type {hf_config.pooling!r} not supported")
assert model_config.pooler_config is not None
model_config.pooler_config.seq_pooling_type = pooling_type
@@ -572,10 +576,13 @@ class LlamaNemotronVLConfig(VerifyAndUpdateConfig):
if pooling is None and hasattr(hf_config, "llm_config"):
pooling = getattr(hf_config.llm_config, "pooling", "avg")
assert isinstance(pooling, str)
pooling_type = pooling_type_map.get(pooling)
if pooling_type is None:
raise ValueError(f"pool_type {pooling!r} not supported")
assert model_config.pooler_config is not None
model_config.pooler_config.seq_pooling_type = pooling_type
@@ -631,7 +638,7 @@ class MambaModelConfig(VerifyAndUpdateConfig):
class NemotronHForCausalLMConfig(VerifyAndUpdateConfig):
DEFAULT_MAMBA_SSM_CACHE_DTYPE = "float32"
DEFAULT_MAMBA_SSM_CACHE_DTYPE: "MambaDType" = "float32"
"""Only `float32` is known to have no accuracy issues by default."""
@classmethod
@@ -643,7 +650,7 @@ class NemotronHForCausalLMConfig(VerifyAndUpdateConfig):
`float32` if not specified.
"""
if cache_config.mamba_ssm_cache_dtype == "auto":
mamba_ssm_cache_dtype = getattr(
mamba_ssm_cache_dtype: MambaDType = getattr(
hf_config, "mamba_ssm_cache_dtype", cls.DEFAULT_MAMBA_SSM_CACHE_DTYPE
)
logger.info(
@@ -729,6 +736,7 @@ class Qwen2ForProcessRewardModelConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
pooler_config = model_config.pooler_config
assert pooler_config is not None
if pooler_config.step_tag_id is None:
pooler_config.step_tag_id = 151651
@@ -738,6 +746,7 @@ class Qwen2ForRewardModelConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
pooler_config = model_config.pooler_config
assert pooler_config is not None
if pooler_config.use_activation is None:
pooler_config.use_activation = False
+10 -6
View File
@@ -66,7 +66,7 @@ class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer):
def dtype(self) -> torch.dtype:
return self.embeddings.patch_embedding.weight.dtype
def forward(
def encode(
self,
pixel_values: torch.Tensor,
grid_thw: torch.Tensor,
@@ -253,7 +253,7 @@ class Cosmos3EdgeVisionModel(nn.Module):
grid_thw: torch.Tensor | list[list[int]],
) -> torch.Tensor:
grid_thw = torch.as_tensor(grid_thw, dtype=torch.int64, device="cpu")
image_embeds = self.encoder(pixel_values.type(self.dtype), grid_thw=grid_thw)
image_embeds = self.encoder.encode(pixel_values.type(self.dtype), grid_thw)
image_embeds = patch_merging_by_param(
image_embeds,
grid_thw,
@@ -315,11 +315,10 @@ class Cosmos3EdgeAttention(NemotronHAttention):
rope_parameters=config.rope_parameters,
)
def forward(
def forward_with_positions(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
**kwargs,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
@@ -367,7 +366,7 @@ class Cosmos3EdgeAttentionDecoderLayer(nn.Module):
hidden_states = self.norm(hidden_states)
else:
hidden_states, residual = self.norm(hidden_states, residual)
hidden_states = self.mixer(positions=positions, hidden_states=hidden_states)
hidden_states = self.mixer.forward_with_positions(positions, hidden_states)
return hidden_states, residual
@@ -562,6 +561,7 @@ class Cosmos3EdgeForConditionalGeneration(
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
assert multimodal_config is not None
self.config = config
self.multimodal_config = multimodal_config
@@ -619,6 +619,8 @@ class Cosmos3EdgeForConditionalGeneration(
image_grid_thw=image_grid_thw,
)
raise AssertionError
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Qwen2_5_VLVideoInputs | None:
@@ -645,6 +647,8 @@ class Cosmos3EdgeForConditionalGeneration(
video_grid_thw=video_grid_thw,
)
raise AssertionError
def _process_image_input(
self, image_input: Qwen2_5_VLImageInputs
) -> tuple[torch.Tensor, ...]:
@@ -684,7 +688,7 @@ class Cosmos3EdgeForConditionalGeneration(
return self._get_image_features(pixel_values_videos, grid_thw)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
modalities = {}
modalities: dict = {}
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is not None:
modalities["image"] = image_input
+10 -4
View File
@@ -3,6 +3,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any
import torch
import torch.nn as nn
@@ -93,8 +94,11 @@ class DbrxExperts(RoutedExperts):
param: nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
param_name: str,
):
shard_id: str,
expert_id: int | None = None,
return_success: Any = False,
) -> Any:
param_name = shard_id
tp_rank = get_tensor_model_parallel_rank()
param_data = param.data
shard_size = self.intermediate_size
@@ -134,6 +138,7 @@ class DbrxExperts(RoutedExperts):
param_data[:] = loaded_weight[:, :, shard]
else:
param_data[:] = loaded_weight
return True if return_success else None
class DbrxMoE(nn.Module):
@@ -409,9 +414,10 @@ class DbrxModel(nn.Module):
if is_pp_missing_parameter(name, self):
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
+4 -3
View File
@@ -10,7 +10,7 @@
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
import math
from collections.abc import Iterable
from collections.abc import Callable, Iterable
from functools import partial
import torch
@@ -73,7 +73,7 @@ class ImageEncoderViT(nn.Module):
mlp_ratio: float = 4.0,
out_chans: int = 256,
qkv_bias: bool = True,
norm_layer: type[nn.Module] = nn.LayerNorm,
norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
act_layer: type[nn.Module] = nn.GELU,
use_abs_pos: bool = True,
use_rel_pos: bool = False,
@@ -205,7 +205,7 @@ class Block(nn.Module):
num_heads: int,
mlp_ratio: float = 4.0,
qkv_bias: bool = True,
norm_layer: type[nn.Module] = nn.LayerNorm,
norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
act_layer: type[nn.Module] = nn.GELU,
use_rel_pos: bool = False,
rel_pos_zero_init: bool = True,
@@ -326,6 +326,7 @@ class RelPosAttention(PluggableLayer):
v = v.view(B, self.num_heads, H * W, -1)
if self.use_rel_pos:
assert rel_h is not None and rel_w is not None
rel_h = rel_h.view(
B, self.num_heads, rel_h.size(1), rel_h.size(2), rel_h.size(3)
)
+3 -3
View File
@@ -15,6 +15,7 @@ from functools import lru_cache
import torch
import torch.nn as nn
import transformers
from transformers.models.qwen2.modeling_qwen2 import Qwen2Model
from vllm.model_executor.custom_op import PluggableLayer
@@ -49,7 +50,6 @@ class CustomQwen2Decoder(PluggableLayer):
super().__init__()
# load
Qwen2Model = transformers.models.qwen2.modeling_qwen2.Qwen2Model
Qwen2Config = transformers.Qwen2Config
# config
@@ -70,11 +70,11 @@ class CustomQwen2Decoder(PluggableLayer):
)
#
self.model = self._create_custom_model(Qwen2Model, config)
self.model = self._create_custom_model(config)
del self.model.embed_tokens
def _create_custom_model(self, Qwen2Model, config):
def _create_custom_model(self, config):
"""Qwen2Model"""
class CustomQwen2ModelInner(Qwen2Model):
+8 -5
View File
@@ -39,6 +39,7 @@ class DeepseekV2Model(nn.Module):
start_layer_id: int = 0,
) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
self.config = vllm_config.speculative_config.draft_model_config.hf_config
quant_config = vllm_config.quant_config
self.vocab_size = self.config.vocab_size
@@ -155,7 +156,7 @@ class DeepseekV2Model(nn.Module):
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
param_name, weight_name, expert_id, expert_shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
@@ -166,7 +167,7 @@ class DeepseekV2Model(nn.Module):
param,
loaded_weight,
name,
shard_id=shard_id,
shard_id=expert_shard_id,
expert_id=expert_id,
)
break
@@ -176,9 +177,10 @@ class DeepseekV2Model(nn.Module):
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
param = params_dict[name]
weight_loader = getattr(
@@ -192,6 +194,7 @@ class DeepseekV2Model(nn.Module):
class EagleDeepseekV3ForCausalLM(DeepseekV3ForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
assert vllm_config.speculative_config is not None
self.config = vllm_config.speculative_config.draft_model_config.hf_config
quant_config = vllm_config.quant_config
target_layer_num = vllm_config.model_config.get_num_layers(
@@ -222,7 +225,7 @@ class EagleDeepseekV3ForCausalLM(DeepseekV3ForCausalLM):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
@@ -159,6 +159,7 @@ class DeepseekV2Eagle3Model(nn.Module):
prefix: str = "",
) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
self.config = vllm_config.speculative_config.draft_model_config.hf_config
self.vocab_size = self.config.vocab_size
@@ -276,6 +277,7 @@ class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
assert vllm_config.speculative_config is not None
self.config = vllm_config.speculative_config.draft_model_config.hf_config
# Ensure draft_vocab_size is set
@@ -318,7 +320,7 @@ class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
+6 -4
View File
@@ -68,6 +68,7 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
super().__init__()
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
self.config = config
quant_config = vllm_config.quant_config
@@ -423,7 +424,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
# with expert_id.
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
param_name, weight_name, expert_id, expert_shard_id = mapping
if weight_name not in chunk_name:
continue
@@ -446,7 +447,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
param,
weight_to_load,
name_mapped,
shard_id=shard_id,
shard_id=expert_shard_id,
expert_id=expert_id,
return_success=True,
)
@@ -467,9 +468,10 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
# According to DeepSeek-V3 Technical Report, MTP modules
# shares embedding layer. We only load the first weights.
+11 -2
View File
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
from vllm.sampling_params import SamplingParams
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
from vllm.transformers_utils.processors.deepseek_ocr import (
BASE_SIZE,
@@ -184,6 +185,7 @@ class NGramPerReqLogitsProcessor(AdapterLogitsProcessor):
)
if ngram_size is None:
return None
assert isinstance(ngram_size, int) and isinstance(window_size, int)
whitelist_token_ids = set(whitelist_token_ids) if whitelist_token_ids else None
return NoRepeatNGramLogitsProcessor(
@@ -301,6 +303,7 @@ class DeepseekOCRMultiModalProcessor(
else:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
processed_outputs = tokenizer(
prompt, add_special_tokens=True, return_tensors="pt"
)
@@ -342,6 +345,7 @@ class DeepseekOCRMultiModalProcessor(
if isinstance(images, ImageEmbeddingItems):
num_image_tokens = images.get_feature_size(item_idx)
else:
assert isinstance(images, ImageProcessorItems)
size = images.get_image_size(item_idx)
num_image_tokens = self.info.get_num_image_tokens(
@@ -459,7 +463,11 @@ class DeepseekOCRForCausalLM(
images_spatial_crop = kwargs.pop("images_spatial_crop", None)
images_crop = kwargs.pop("images_crop", None)
if pixel_values is None or torch.sum(pixel_values).item() == 0:
if pixel_values is None:
return None
assert isinstance(pixel_values, torch.Tensor)
assert images_crop is None or isinstance(images_crop, torch.Tensor)
if torch.sum(pixel_values).item() == 0:
return None
# Use actual tensor spatial dim instead of hardcoded
@@ -675,7 +683,8 @@ class DeepseekOCRForCausalLM(
num_input_tokens = global_input_side**2
if is_tiled:
num_patches = image_spatial_crop.prod(dim=-1)
assert image_spatial_crop is not None
num_patches = int(image_spatial_crop.prod().item())
num_input_tokens += num_patches * (local_input_side**2)
global_output_token = self.global_image_output_token
+4 -1
View File
@@ -47,6 +47,7 @@ from vllm.multimodal.processing import (
)
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
from vllm.transformers_utils.processors.deepseek_ocr import (
BASE_SIZE,
@@ -173,6 +174,7 @@ class DeepseekOCR2MultiModalProcessor(
else:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
processed_outputs = tokenizer(
prompt, add_special_tokens=True, return_tensors="pt"
)
@@ -214,6 +216,7 @@ class DeepseekOCR2MultiModalProcessor(
if isinstance(images, ImageEmbeddingItems):
num_image_tokens = images.get_feature_size(item_idx)
else:
assert isinstance(images, ImageProcessorItems)
size = images.get_image_size(item_idx)
num_image_tokens = self.info.get_num_image_tokens(
@@ -284,7 +287,7 @@ class DeepseekOCR2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Support
patch_size=16,
qkv_bias=True,
use_rel_pos=True,
global_attn_indexes=[2, 5, 8, 11],
global_attn_indexes=(2, 5, 8, 11),
window_size=14,
out_chans=256,
last_conv_output=896,
+13 -6
View File
@@ -626,7 +626,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
def forward(self): ...
def get_attn_backend(self) -> AttentionBackend:
def get_attn_backend(self) -> type[AttentionBackend]:
return DeepseekV32IndexerBackend
@@ -681,6 +681,7 @@ class Indexer(nn.Module):
# NOTE: (zyongye) we use fp8 naive cache,
# where we store value in fp8 and scale in fp32
# per self.quant_block_size element
assert cache_config is not None
self.k_cache = DeepseekV32IndexerCache(
head_dim=self.head_dim + self.head_dim // self.quant_block_size * 4,
dtype=torch.uint8,
@@ -1107,6 +1108,10 @@ class DeepseekV2MLAAttention(nn.Module):
)
if self.is_v32 and (not _skip_topk or is_mtp_layer):
assert q_lora_rank is not None
assert cache_config is not None
self.indexer_rope_emb: nn.Module | None
self.indexer: Indexer | None
self.indexer_rope_emb = get_rope(
qk_rope_head_dim,
max_position=max_position_embeddings,
@@ -1218,6 +1223,7 @@ class DeepseekV2DecoderLayer(nn.Module):
self.use_mha = use_mha
attn_cls: typing.Any
if use_mha:
attn_cls = DeepseekAttention
elif model_config.use_mla:
@@ -1508,7 +1514,7 @@ class DeepseekV2Model(nn.Module):
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
)
stacked_params_mapping = [
stacked_params_mapping: list[tuple[str, str, int | str]] = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
@@ -1677,7 +1683,7 @@ class DeepseekV2Model(nn.Module):
# param and delegate to its expert-aware weight_loader
# with expert_id.
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
param_name, weight_name, expert_id, expert_shard_id = mapping
if weight_name not in chunk_name:
continue
@@ -1703,7 +1709,7 @@ class DeepseekV2Model(nn.Module):
param,
weight_to_load,
name_mapped,
shard_id=shard_id,
shard_id=expert_shard_id,
expert_id=expert_id,
return_success=True,
)
@@ -1725,9 +1731,10 @@ class DeepseekV2Model(nn.Module):
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
if is_pp_missing_parameter(name, self):
continue
+6 -1
View File
@@ -43,6 +43,7 @@ from vllm.multimodal.processing.processor import (
)
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.configs.deepseek_vl2 import (
DeepseekVLV2Config,
MlpProjectorConfig,
@@ -244,6 +245,7 @@ class DeepseekVL2MultiModalProcessor(
) -> BatchFeature:
if not mm_data:
tokenizer = self.info.get_tokenizer()
assert isinstance(tokenizer, HfTokenizer)
return tokenizer(prompt, add_special_tokens=True, return_tensors="pt")
processed_outputs = super()._call_hf_processor(
@@ -291,6 +293,7 @@ class DeepseekVL2MultiModalProcessor(
if isinstance(images, ImageEmbeddingItems):
num_image_tokens = images.get_feature_size(item_idx)
else:
assert isinstance(images, ImageProcessorItems)
image_size = images.get_image_size(item_idx)
num_image_tokens = self.info.get_num_image_tokens(
@@ -406,7 +409,9 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
return parent, names[-1]
# patch for timm ViT instance to support tensor parallel
def patch_vit_for_tp(self, vit: torch.nn.Module, quant_config: QuantizationConfig):
def patch_vit_for_tp(
self, vit: torch.nn.Module, quant_config: QuantizationConfig | None
):
try:
import timm
except ImportError as e:
@@ -32,6 +32,7 @@ from vllm.distributed.parallel_state import get_tp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
)
@@ -195,8 +196,10 @@ class DiffusionGemmaForConditionalGeneration(
# ---- Vision tower ----
vision_config = getattr(config, "vision_config", None)
self.embed_vision: Gemma4MultimodalEmbedder | None
if vision_config is not None:
quant_config = vllm_config.quant_config
tower_quant: QuantizationConfig | None
if quant_config and quant_config.get_name() in [
"bitsandbytes",
"torchao",
@@ -1048,7 +1051,7 @@ class DiffusionSampler:
sampler: Any,
diffusion_config: Any,
vocab_size: int,
diffusion_states: DiffusionGemmaRequestStates | None = None,
diffusion_states: DiffusionGemmaRequestStates,
*,
confidence_threshold: float,
t_min: float,
+14 -7
View File
@@ -149,7 +149,7 @@ class DotsOCRProcessingInfo(Qwen2VLProcessingInfo):
self,
**kwargs: object,
) -> Qwen2VLProcessor:
self.get_tokenizer().image_token = IMAGE_TOKEN # Ensure image token is set
setattr(self.get_tokenizer(), "image_token", IMAGE_TOKEN) # noqa: B010
processor = self.ctx.get_hf_processor(
Qwen2VLProcessor,
**kwargs,
@@ -430,7 +430,7 @@ class DotsVisionBlock(nn.Module):
*,
cu_seqlens: torch.Tensor,
rotary_pos_emb: torch.Tensor,
max_seqlen: int | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
hidden_states = hidden_states + self.attn(
self.norm1(hidden_states),
@@ -483,6 +483,7 @@ class DotsVisionTransformer(nn.Module):
)
if require_post_norm is None:
require_post_norm = len(self.blocks) == config.num_hidden_layers
self.post_trunk_norm: RMSNorm | None
if require_post_norm and self.config.post_norm:
self.post_trunk_norm = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)
else:
@@ -536,7 +537,7 @@ class DotsVisionTransformer(nn.Module):
rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
return rotary_pos_emb
def compute_attn_mask_seqlen(self, cu_seqlens: torch.Tensor) -> int | None:
def compute_attn_mask_seqlen(self, cu_seqlens: torch.Tensor) -> torch.Tensor | None:
max_seqlen = None
if self.attn_backend in {
AttentionBackendEnum.FLASH_ATTN,
@@ -552,15 +553,17 @@ class DotsVisionTransformer(nn.Module):
rotary_pos_emb = self.rot_pos_emb(grid_thw)
# Convert grid_thw to tensor (always expecting list format now)
grid_thw = torch.tensor(grid_thw, device=hidden_states.device, dtype=torch.long)
grid_thw_tensor = torch.tensor(
grid_thw, device=hidden_states.device, dtype=torch.long
)
hidden_states = hidden_states.to(self.dtype)
hidden_states = self.patch_embed(hidden_states, grid_thw)
hidden_states = self.patch_embed(hidden_states, grid_thw_tensor)
cu_seqlens = torch.repeat_interleave(
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
grid_thw_tensor[:, 1] * grid_thw_tensor[:, 2], grid_thw_tensor[:, 0]
).cumsum(
dim=0,
dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32,
dtype=grid_thw_tensor.dtype if torch.jit.is_tracing() else torch.int32,
)
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
@@ -620,6 +623,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|img|><|imgpad|><|endofimg|>"
return None
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -627,6 +631,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
self.config: DotsOCRConfig = vllm_config.model_config.hf_config
self.quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
assert multimodal_config is not None
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
if isinstance(self.config.vision_config, dict):
vision_config = DotsVisionConfig(**self.config.vision_config)
@@ -677,6 +682,8 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
image_grid_thw=image_grid_thw,
)
raise AssertionError
def _process_image_input(
self, image_input: DotsOCRImageInputs
) -> tuple[torch.Tensor, ...]:
+2 -1
View File
@@ -118,8 +118,9 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
width: int,
height: int,
num_images: int,
overrides: ImageDummyOptions | None = None,
overrides: BaseDummyOptions | None = None,
) -> list[Image.Image]:
assert overrides is None or isinstance(overrides, ImageDummyOptions)
if num_images == 0:
return []
if overrides: