[Mypy Fix] Mypy fix for "vllm/model_executor/models/[cC][dD]" (#52003)
This commit is contained in:
@@ -111,10 +111,7 @@ SEPARATE_GROUPS = [
|
||||
"tests/v1/worker",
|
||||
]
|
||||
|
||||
# TODO(woosuk): Include the code from Megatron and HuggingFace.
|
||||
EXCLUDE = [
|
||||
r"vllm/model_executor/models/[cC]",
|
||||
r"vllm/model_executor/models/[dD]",
|
||||
r"vllm/model_executor/models/[eE]",
|
||||
r"vllm/model_executor/models/[fF]",
|
||||
r"vllm/model_executor/models/[gG]",
|
||||
|
||||
@@ -142,6 +142,7 @@ class CLIPProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
pooler_config = self.ctx.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
assert pooler_config.seq_pooling_type is not None
|
||||
|
||||
return get_num_selected_vision_tokens(
|
||||
vision_encoder_info.get_num_image_tokens(
|
||||
@@ -854,6 +855,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
|
||||
feature_select_strategy: VisionFeatureSelectStrategy | None = None,
|
||||
) -> torch.Tensor:
|
||||
if feature_select_strategy is None:
|
||||
assert self.pooler_config.seq_pooling_type is not None
|
||||
feature_select_strategy = _get_vision_feature_select_strategy(
|
||||
self.pooler_config.seq_pooling_type
|
||||
)
|
||||
@@ -961,6 +963,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
|
||||
raise RuntimeError("PP is not supported for this model")
|
||||
|
||||
# Multimodal inputs
|
||||
assert inputs_embeds is not None
|
||||
if not self._is_text_input:
|
||||
return inputs_embeds
|
||||
|
||||
|
||||
@@ -178,6 +178,7 @@ class Cohere2MoeAttention(nn.Module):
|
||||
self.max_position_embeddings = getattr(
|
||||
config, "model_max_length", None
|
||||
) or getattr(config, "max_position_embeddings", 8192)
|
||||
assert isinstance(self.max_position_embeddings, int)
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
@@ -283,6 +284,7 @@ class Cohere2Moe(nn.Module):
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.shared_experts: Cohere2MoeMLP | None
|
||||
if hasattr(config, "num_shared_experts") and config.num_shared_experts > 0:
|
||||
self.shared_experts = Cohere2MoeMLP(
|
||||
config=config,
|
||||
|
||||
@@ -416,7 +416,7 @@ class Cohere2VisionForConditionalGeneration(
|
||||
)
|
||||
|
||||
def _patch_quant_config(
|
||||
self, config: PretrainedConfig, quant_config: QuantizationConfig
|
||||
self, config: PretrainedConfig, quant_config: QuantizationConfig | None
|
||||
):
|
||||
# the awq models from OpenGVLab missing `modules_to_not_convert`
|
||||
# patch the quant_config to add `modules_to_not_convert` back
|
||||
|
||||
@@ -55,6 +55,7 @@ from vllm.transformers_utils.processors.cohere_asr import (
|
||||
CohereASRFeatureExtractor,
|
||||
CohereASRProcessor,
|
||||
)
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionType,
|
||||
)
|
||||
@@ -183,6 +184,7 @@ class CohereASRAttention(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
@@ -241,7 +243,7 @@ class CohereASRCrossAttention(CohereASRAttention):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor | None,
|
||||
encoder_hidden_states: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
q, _ = self.q_proj(hidden_states)
|
||||
|
||||
@@ -837,7 +839,7 @@ class CausalConv1D(nn.Conv1d):
|
||||
out_channels: int,
|
||||
kernel_size: int,
|
||||
stride: int = 1,
|
||||
padding: str | int = 0,
|
||||
padding: str | int | list[int] | None = 0,
|
||||
dilation: int = 1,
|
||||
groups: int = 1,
|
||||
bias: bool = True,
|
||||
@@ -845,6 +847,8 @@ class CausalConv1D(nn.Conv1d):
|
||||
device=None,
|
||||
dtype=None,
|
||||
) -> None:
|
||||
self._left_padding: int
|
||||
self._right_padding: int
|
||||
if padding is None:
|
||||
self._left_padding = kernel_size - 1
|
||||
self._right_padding = stride - 1
|
||||
@@ -903,7 +907,7 @@ class ConformerConvolution(nn.Module):
|
||||
d_model: int,
|
||||
kernel_size: int,
|
||||
norm_type: str = "batch_norm",
|
||||
conv_context_size: int | None = None,
|
||||
conv_context_size: int | list[int] | None = None,
|
||||
pointwise_activation: str = "glu_",
|
||||
use_bias: bool = True,
|
||||
) -> None:
|
||||
@@ -1158,6 +1162,7 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention):
|
||||
q, k, v = self.forward_qkv(query, key, value)
|
||||
q = q.transpose(1, 2) # (batch, time1, head, d_k)
|
||||
|
||||
assert pos_emb is not None
|
||||
n_batch_pos = pos_emb.size(0)
|
||||
p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k)
|
||||
p = p.transpose(1, 2) # (batch, head, time1, d_k)
|
||||
@@ -1213,7 +1218,7 @@ class ConformerLayer(torch.nn.Module):
|
||||
n_heads: int = 4,
|
||||
conv_kernel_size: int = 31,
|
||||
conv_norm_type: str = "batch_norm",
|
||||
conv_context_size: int | None = None,
|
||||
conv_context_size: int | list[int] | None = None,
|
||||
pos_bias_u: nn.Parameter | torch.Tensor | None = None,
|
||||
pos_bias_v: nn.Parameter | torch.Tensor | None = None,
|
||||
att_context_size: list[int] | None = None,
|
||||
@@ -1382,6 +1387,7 @@ class ConformerEncoder(nn.Module):
|
||||
conv_kernel_size=conv_kernel_size,
|
||||
)
|
||||
|
||||
self.xscale: float | None
|
||||
if xscaling:
|
||||
self.xscale = math.sqrt(d_model)
|
||||
else:
|
||||
@@ -1633,9 +1639,12 @@ class ConformerEncoder(nn.Module):
|
||||
) -> tuple[list[list[int]], list[int], list[float], list[int]]:
|
||||
# convert att_context_size to a standard list of lists
|
||||
if att_context_size:
|
||||
att_context_size_all = list(att_context_size)
|
||||
if isinstance(att_context_size_all[0], int):
|
||||
att_context_size_all = [att_context_size_all]
|
||||
if is_list_of(att_context_size, int, check="all"):
|
||||
att_context_size_all = [att_context_size]
|
||||
elif is_list_of(att_context_size, list, check="all"):
|
||||
att_context_size_all = att_context_size
|
||||
else:
|
||||
raise ValueError("att_context_size cannot mix nested and flat values")
|
||||
for i, att_cs in enumerate(att_context_size_all):
|
||||
if att_context_style == "chunked_limited":
|
||||
if att_cs[0] > 0 and att_cs[0] % (att_cs[1] + 1) > 0:
|
||||
@@ -1684,6 +1693,7 @@ class ConformerEncoder(nn.Module):
|
||||
if conv_context_size == "causal":
|
||||
conv_context_size = [conv_kernel_size - 1, 0]
|
||||
else:
|
||||
assert isinstance(conv_context_size, list)
|
||||
total = conv_context_size[0] + conv_context_size[1] + 1
|
||||
if total != conv_kernel_size:
|
||||
raise ValueError(
|
||||
@@ -1940,7 +1950,7 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing
|
||||
):
|
||||
if mm_data:
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
mm_data = dict(audio=mm_data.pop("audios"))
|
||||
mm_data = dict(audio=dict(mm_data).pop("audios"))
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
@@ -2045,7 +2055,6 @@ class CohereAsrForConditionalGeneration(
|
||||
# (which is different from "_") and encode("▁ABC") ignores the first token
|
||||
# so the prompt_text is unreliable. However, prompt_token_ids can be used
|
||||
# to get prompt_text but it wont have the first token "▁".
|
||||
prompt_text = None
|
||||
prompt_token_ids = cls._get_default_prompt_token_ids(
|
||||
tokenizer,
|
||||
model_config,
|
||||
@@ -2053,7 +2062,6 @@ class CohereAsrForConditionalGeneration(
|
||||
)
|
||||
|
||||
return TokensPrompt(
|
||||
prompt=prompt_text,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
multi_modal_data={"audio": (audio, stt_config.sample_rate)},
|
||||
)
|
||||
@@ -2199,7 +2207,9 @@ class CohereAsrForConditionalGeneration(
|
||||
audio_input, seq_lens = self._parse_and_validate_audio_input(**kwargs)
|
||||
|
||||
if hasattr(audio_input, "input_features"):
|
||||
out = self.model.get_encoder_outputs(audio_input["input_features"])
|
||||
out = self.model.get_encoder_outputs(
|
||||
audio_input["input_features"], seq_lens
|
||||
)
|
||||
else:
|
||||
out = self.model.get_encoder_outputs(audio_input, seq_lens)
|
||||
|
||||
@@ -2219,12 +2229,17 @@ class CohereAsrForConditionalGeneration(
|
||||
f"Incorrect type of audio features. Got type: {type(input_features)}"
|
||||
)
|
||||
|
||||
if not isinstance(length, torch.Tensor):
|
||||
raise ValueError("Audio feature lengths must be a tensor.")
|
||||
|
||||
if isinstance(input_features, torch.Tensor):
|
||||
seq_lens = length.reshape(-1)
|
||||
else:
|
||||
if not is_list_of(input_features, torch.Tensor, check="all"):
|
||||
raise ValueError("All audio features must be tensors.")
|
||||
input_features = [
|
||||
feat.to(self.dtype).squeeze(0).transpose(1, 0)
|
||||
for feat in input_features
|
||||
feature.to(self.dtype).squeeze(0).transpose(1, 0)
|
||||
for feature in input_features
|
||||
]
|
||||
seq_lens = length.reshape(-1)
|
||||
input_features = torch.nn.utils.rnn.pad_sequence(
|
||||
|
||||
@@ -58,6 +58,7 @@ class CohereEagleModel(nn.Module):
|
||||
start_layer_id: int = 0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.quant_config = get_draft_quant_config(vllm_config)
|
||||
|
||||
@@ -137,6 +138,7 @@ class CohereEagleModel(nn.Module):
|
||||
class EagleCohereForCausalLM(CohereForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
# Flags checked by the speculative proposer to decide whether to share
|
||||
# embed_tokens / lm_head with the target model. Cohere EAGLE checkpoints
|
||||
@@ -160,7 +162,7 @@ class EagleCohereForCausalLM(CohereForCausalLM):
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.transformers_utils.configs.colmodernvbert import ColModernVBertConfig
|
||||
|
||||
from .interfaces import (
|
||||
@@ -161,6 +162,7 @@ class ColModernVBertMultiModalProcessor(
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
text_encoding = tokenizer(
|
||||
prompt,
|
||||
return_tensors="pt",
|
||||
|
||||
@@ -25,6 +25,7 @@ from transformers import BatchFeature, PaliGemmaProcessor
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
|
||||
from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
@@ -232,7 +233,9 @@ class ColPaliModel(
|
||||
loaded.add(f"custom_text_proj.{param_name}")
|
||||
|
||||
# Update pooler projector for the lazy-creation path
|
||||
self.pooler.head.projector = self.custom_text_proj
|
||||
head = self.pooler.head
|
||||
assert isinstance(head, TokenPoolerHead)
|
||||
head.projector = self.custom_text_proj
|
||||
|
||||
# Mark pooler projector params as loaded
|
||||
if hasattr(self, "pooler") and hasattr(self.pooler, "head"):
|
||||
|
||||
@@ -154,6 +154,7 @@ class CohereAttention(nn.Module):
|
||||
self.max_position_embeddings = getattr(
|
||||
config, "model_max_length", None
|
||||
) or getattr(config, "max_position_embeddings", 8192)
|
||||
assert isinstance(self.max_position_embeddings, int)
|
||||
self.use_qk_norm = getattr(config, "use_qk_norm", False)
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
|
||||
@@ -9,6 +9,7 @@ if TYPE_CHECKING:
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.config.cache import MambaDType
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -452,6 +453,7 @@ class JambaForSequenceClassificationConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
pooler_config = model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
if pooler_config.use_activation is None:
|
||||
pooler_config.use_activation = False
|
||||
|
||||
@@ -518,6 +520,7 @@ class JinaVLForSequenceClassificationConfig(VerifyAndUpdateConfig):
|
||||
config = model_config.hf_config
|
||||
config.num_labels = 1
|
||||
pooler_config = model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
if pooler_config.logit_mean is None:
|
||||
pooler_config.logit_mean = 2.65
|
||||
|
||||
@@ -540,6 +543,7 @@ class LlamaBidirectionalConfig(VerifyAndUpdateConfig):
|
||||
if pooling_type is None:
|
||||
raise ValueError(f"pool_type {hf_config.pooling!r} not supported")
|
||||
|
||||
assert model_config.pooler_config is not None
|
||||
model_config.pooler_config.seq_pooling_type = pooling_type
|
||||
|
||||
|
||||
@@ -572,10 +576,13 @@ class LlamaNemotronVLConfig(VerifyAndUpdateConfig):
|
||||
if pooling is None and hasattr(hf_config, "llm_config"):
|
||||
pooling = getattr(hf_config.llm_config, "pooling", "avg")
|
||||
|
||||
assert isinstance(pooling, str)
|
||||
|
||||
pooling_type = pooling_type_map.get(pooling)
|
||||
if pooling_type is None:
|
||||
raise ValueError(f"pool_type {pooling!r} not supported")
|
||||
|
||||
assert model_config.pooler_config is not None
|
||||
model_config.pooler_config.seq_pooling_type = pooling_type
|
||||
|
||||
|
||||
@@ -631,7 +638,7 @@ class MambaModelConfig(VerifyAndUpdateConfig):
|
||||
|
||||
|
||||
class NemotronHForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
DEFAULT_MAMBA_SSM_CACHE_DTYPE = "float32"
|
||||
DEFAULT_MAMBA_SSM_CACHE_DTYPE: "MambaDType" = "float32"
|
||||
"""Only `float32` is known to have no accuracy issues by default."""
|
||||
|
||||
@classmethod
|
||||
@@ -643,7 +650,7 @@ class NemotronHForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
`float32` if not specified.
|
||||
"""
|
||||
if cache_config.mamba_ssm_cache_dtype == "auto":
|
||||
mamba_ssm_cache_dtype = getattr(
|
||||
mamba_ssm_cache_dtype: MambaDType = getattr(
|
||||
hf_config, "mamba_ssm_cache_dtype", cls.DEFAULT_MAMBA_SSM_CACHE_DTYPE
|
||||
)
|
||||
logger.info(
|
||||
@@ -729,6 +736,7 @@ class Qwen2ForProcessRewardModelConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
pooler_config = model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
if pooler_config.step_tag_id is None:
|
||||
pooler_config.step_tag_id = 151651
|
||||
@@ -738,6 +746,7 @@ class Qwen2ForRewardModelConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
pooler_config = model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
if pooler_config.use_activation is None:
|
||||
pooler_config.use_activation = False
|
||||
|
||||
@@ -66,7 +66,7 @@ class Cosmos3EdgeVisionEncoder(Siglip2VisionTransformer):
|
||||
def dtype(self) -> torch.dtype:
|
||||
return self.embeddings.patch_embedding.weight.dtype
|
||||
|
||||
def forward(
|
||||
def encode(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
grid_thw: torch.Tensor,
|
||||
@@ -253,7 +253,7 @@ class Cosmos3EdgeVisionModel(nn.Module):
|
||||
grid_thw: torch.Tensor | list[list[int]],
|
||||
) -> torch.Tensor:
|
||||
grid_thw = torch.as_tensor(grid_thw, dtype=torch.int64, device="cpu")
|
||||
image_embeds = self.encoder(pixel_values.type(self.dtype), grid_thw=grid_thw)
|
||||
image_embeds = self.encoder.encode(pixel_values.type(self.dtype), grid_thw)
|
||||
image_embeds = patch_merging_by_param(
|
||||
image_embeds,
|
||||
grid_thw,
|
||||
@@ -315,11 +315,10 @@ class Cosmos3EdgeAttention(NemotronHAttention):
|
||||
rope_parameters=config.rope_parameters,
|
||||
)
|
||||
|
||||
def forward(
|
||||
def forward_with_positions(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
@@ -367,7 +366,7 @@ class Cosmos3EdgeAttentionDecoderLayer(nn.Module):
|
||||
hidden_states = self.norm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.norm(hidden_states, residual)
|
||||
hidden_states = self.mixer(positions=positions, hidden_states=hidden_states)
|
||||
hidden_states = self.mixer.forward_with_positions(positions, hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -562,6 +561,7 @@ class Cosmos3EdgeForConditionalGeneration(
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
assert multimodal_config is not None
|
||||
|
||||
self.config = config
|
||||
self.multimodal_config = multimodal_config
|
||||
@@ -619,6 +619,8 @@ class Cosmos3EdgeForConditionalGeneration(
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
|
||||
raise AssertionError
|
||||
|
||||
def _parse_and_validate_video_input(
|
||||
self, **kwargs: object
|
||||
) -> Qwen2_5_VLVideoInputs | None:
|
||||
@@ -645,6 +647,8 @@ class Cosmos3EdgeForConditionalGeneration(
|
||||
video_grid_thw=video_grid_thw,
|
||||
)
|
||||
|
||||
raise AssertionError
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: Qwen2_5_VLImageInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
@@ -684,7 +688,7 @@ class Cosmos3EdgeForConditionalGeneration(
|
||||
return self._get_image_features(pixel_values_videos, grid_thw)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
modalities = {}
|
||||
modalities: dict = {}
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
if image_input is not None:
|
||||
modalities["image"] = image_input
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -93,8 +94,11 @@ class DbrxExperts(RoutedExperts):
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
param_name: str,
|
||||
):
|
||||
shard_id: str,
|
||||
expert_id: int | None = None,
|
||||
return_success: Any = False,
|
||||
) -> Any:
|
||||
param_name = shard_id
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
param_data = param.data
|
||||
shard_size = self.intermediate_size
|
||||
@@ -134,6 +138,7 @@ class DbrxExperts(RoutedExperts):
|
||||
param_data[:] = loaded_weight[:, :, shard]
|
||||
else:
|
||||
param_data[:] = loaded_weight
|
||||
return True if return_success else None
|
||||
|
||||
|
||||
class DbrxMoE(nn.Module):
|
||||
@@ -409,9 +414,10 @@ class DbrxModel(nn.Module):
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
# This source code is licensed under the license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
import math
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Callable, Iterable
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
@@ -73,7 +73,7 @@ class ImageEncoderViT(nn.Module):
|
||||
mlp_ratio: float = 4.0,
|
||||
out_chans: int = 256,
|
||||
qkv_bias: bool = True,
|
||||
norm_layer: type[nn.Module] = nn.LayerNorm,
|
||||
norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
|
||||
act_layer: type[nn.Module] = nn.GELU,
|
||||
use_abs_pos: bool = True,
|
||||
use_rel_pos: bool = False,
|
||||
@@ -205,7 +205,7 @@ class Block(nn.Module):
|
||||
num_heads: int,
|
||||
mlp_ratio: float = 4.0,
|
||||
qkv_bias: bool = True,
|
||||
norm_layer: type[nn.Module] = nn.LayerNorm,
|
||||
norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
|
||||
act_layer: type[nn.Module] = nn.GELU,
|
||||
use_rel_pos: bool = False,
|
||||
rel_pos_zero_init: bool = True,
|
||||
@@ -326,6 +326,7 @@ class RelPosAttention(PluggableLayer):
|
||||
v = v.view(B, self.num_heads, H * W, -1)
|
||||
|
||||
if self.use_rel_pos:
|
||||
assert rel_h is not None and rel_w is not None
|
||||
rel_h = rel_h.view(
|
||||
B, self.num_heads, rel_h.size(1), rel_h.size(2), rel_h.size(3)
|
||||
)
|
||||
|
||||
@@ -15,6 +15,7 @@ from functools import lru_cache
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import transformers
|
||||
from transformers.models.qwen2.modeling_qwen2 import Qwen2Model
|
||||
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
|
||||
@@ -49,7 +50,6 @@ class CustomQwen2Decoder(PluggableLayer):
|
||||
super().__init__()
|
||||
|
||||
# load
|
||||
Qwen2Model = transformers.models.qwen2.modeling_qwen2.Qwen2Model
|
||||
Qwen2Config = transformers.Qwen2Config
|
||||
|
||||
# config
|
||||
@@ -70,11 +70,11 @@ class CustomQwen2Decoder(PluggableLayer):
|
||||
)
|
||||
|
||||
#
|
||||
self.model = self._create_custom_model(Qwen2Model, config)
|
||||
self.model = self._create_custom_model(config)
|
||||
|
||||
del self.model.embed_tokens
|
||||
|
||||
def _create_custom_model(self, Qwen2Model, config):
|
||||
def _create_custom_model(self, config):
|
||||
"""Qwen2Model"""
|
||||
|
||||
class CustomQwen2ModelInner(Qwen2Model):
|
||||
|
||||
@@ -39,6 +39,7 @@ class DeepseekV2Model(nn.Module):
|
||||
start_layer_id: int = 0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.vocab_size = self.config.vocab_size
|
||||
@@ -155,7 +156,7 @@ class DeepseekV2Model(nn.Module):
|
||||
break
|
||||
else:
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
param_name, weight_name, expert_id, expert_shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
@@ -166,7 +167,7 @@ class DeepseekV2Model(nn.Module):
|
||||
param,
|
||||
loaded_weight,
|
||||
name,
|
||||
shard_id=shard_id,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
break
|
||||
@@ -176,9 +177,10 @@ class DeepseekV2Model(nn.Module):
|
||||
continue
|
||||
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
@@ -192,6 +194,7 @@ class DeepseekV2Model(nn.Module):
|
||||
class EagleDeepseekV3ForCausalLM(DeepseekV3ForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
@@ -222,7 +225,7 @@ class EagleDeepseekV3ForCausalLM(DeepseekV3ForCausalLM):
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
|
||||
@@ -159,6 +159,7 @@ class DeepseekV2Eagle3Model(nn.Module):
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.vocab_size = self.config.vocab_size
|
||||
|
||||
@@ -276,6 +277,7 @@ class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
assert vllm_config.speculative_config is not None
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
|
||||
# Ensure draft_vocab_size is set
|
||||
@@ -318,7 +320,7 @@ class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
|
||||
) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
|
||||
@@ -68,6 +68,7 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
|
||||
super().__init__()
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
@@ -423,7 +424,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
# with expert_id.
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
param_name, weight_name, expert_id, expert_shard_id = mapping
|
||||
if weight_name not in chunk_name:
|
||||
continue
|
||||
|
||||
@@ -446,7 +447,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
param,
|
||||
weight_to_load,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
@@ -467,9 +468,10 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
|
||||
# According to DeepSeek-V3 Technical Report, MTP modules
|
||||
# shares embedding layer. We only load the first weights.
|
||||
|
||||
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import (
|
||||
BASE_SIZE,
|
||||
@@ -184,6 +185,7 @@ class NGramPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
)
|
||||
if ngram_size is None:
|
||||
return None
|
||||
assert isinstance(ngram_size, int) and isinstance(window_size, int)
|
||||
|
||||
whitelist_token_ids = set(whitelist_token_ids) if whitelist_token_ids else None
|
||||
return NoRepeatNGramLogitsProcessor(
|
||||
@@ -301,6 +303,7 @@ class DeepseekOCRMultiModalProcessor(
|
||||
|
||||
else:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
processed_outputs = tokenizer(
|
||||
prompt, add_special_tokens=True, return_tensors="pt"
|
||||
)
|
||||
@@ -342,6 +345,7 @@ class DeepseekOCRMultiModalProcessor(
|
||||
if isinstance(images, ImageEmbeddingItems):
|
||||
num_image_tokens = images.get_feature_size(item_idx)
|
||||
else:
|
||||
assert isinstance(images, ImageProcessorItems)
|
||||
size = images.get_image_size(item_idx)
|
||||
|
||||
num_image_tokens = self.info.get_num_image_tokens(
|
||||
@@ -459,7 +463,11 @@ class DeepseekOCRForCausalLM(
|
||||
images_spatial_crop = kwargs.pop("images_spatial_crop", None)
|
||||
images_crop = kwargs.pop("images_crop", None)
|
||||
|
||||
if pixel_values is None or torch.sum(pixel_values).item() == 0:
|
||||
if pixel_values is None:
|
||||
return None
|
||||
assert isinstance(pixel_values, torch.Tensor)
|
||||
assert images_crop is None or isinstance(images_crop, torch.Tensor)
|
||||
if torch.sum(pixel_values).item() == 0:
|
||||
return None
|
||||
|
||||
# Use actual tensor spatial dim instead of hardcoded
|
||||
@@ -675,7 +683,8 @@ class DeepseekOCRForCausalLM(
|
||||
num_input_tokens = global_input_side**2
|
||||
|
||||
if is_tiled:
|
||||
num_patches = image_spatial_crop.prod(dim=-1)
|
||||
assert image_spatial_crop is not None
|
||||
num_patches = int(image_spatial_crop.prod().item())
|
||||
num_input_tokens += num_patches * (local_input_side**2)
|
||||
|
||||
global_output_token = self.global_image_output_token
|
||||
|
||||
@@ -47,6 +47,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import (
|
||||
BASE_SIZE,
|
||||
@@ -173,6 +174,7 @@ class DeepseekOCR2MultiModalProcessor(
|
||||
|
||||
else:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
processed_outputs = tokenizer(
|
||||
prompt, add_special_tokens=True, return_tensors="pt"
|
||||
)
|
||||
@@ -214,6 +216,7 @@ class DeepseekOCR2MultiModalProcessor(
|
||||
if isinstance(images, ImageEmbeddingItems):
|
||||
num_image_tokens = images.get_feature_size(item_idx)
|
||||
else:
|
||||
assert isinstance(images, ImageProcessorItems)
|
||||
size = images.get_image_size(item_idx)
|
||||
|
||||
num_image_tokens = self.info.get_num_image_tokens(
|
||||
@@ -284,7 +287,7 @@ class DeepseekOCR2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Support
|
||||
patch_size=16,
|
||||
qkv_bias=True,
|
||||
use_rel_pos=True,
|
||||
global_attn_indexes=[2, 5, 8, 11],
|
||||
global_attn_indexes=(2, 5, 8, 11),
|
||||
window_size=14,
|
||||
out_chans=256,
|
||||
last_conv_output=896,
|
||||
|
||||
@@ -626,7 +626,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
|
||||
def forward(self): ...
|
||||
|
||||
def get_attn_backend(self) -> AttentionBackend:
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
return DeepseekV32IndexerBackend
|
||||
|
||||
|
||||
@@ -681,6 +681,7 @@ class Indexer(nn.Module):
|
||||
# NOTE: (zyongye) we use fp8 naive cache,
|
||||
# where we store value in fp8 and scale in fp32
|
||||
# per self.quant_block_size element
|
||||
assert cache_config is not None
|
||||
self.k_cache = DeepseekV32IndexerCache(
|
||||
head_dim=self.head_dim + self.head_dim // self.quant_block_size * 4,
|
||||
dtype=torch.uint8,
|
||||
@@ -1107,6 +1108,10 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
)
|
||||
|
||||
if self.is_v32 and (not _skip_topk or is_mtp_layer):
|
||||
assert q_lora_rank is not None
|
||||
assert cache_config is not None
|
||||
self.indexer_rope_emb: nn.Module | None
|
||||
self.indexer: Indexer | None
|
||||
self.indexer_rope_emb = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
@@ -1218,6 +1223,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
|
||||
self.use_mha = use_mha
|
||||
|
||||
attn_cls: typing.Any
|
||||
if use_mha:
|
||||
attn_cls = DeepseekAttention
|
||||
elif model_config.use_mla:
|
||||
@@ -1508,7 +1514,7 @@ class DeepseekV2Model(nn.Module):
|
||||
rocm_aiter_moe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
stacked_params_mapping = [
|
||||
stacked_params_mapping: list[tuple[str, str, int | str]] = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
@@ -1677,7 +1683,7 @@ class DeepseekV2Model(nn.Module):
|
||||
# param and delegate to its expert-aware weight_loader
|
||||
# with expert_id.
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
param_name, weight_name, expert_id, expert_shard_id = mapping
|
||||
if weight_name not in chunk_name:
|
||||
continue
|
||||
|
||||
@@ -1703,7 +1709,7 @@ class DeepseekV2Model(nn.Module):
|
||||
param,
|
||||
weight_to_load,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
shard_id=expert_shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
@@ -1725,9 +1731,10 @@ class DeepseekV2Model(nn.Module):
|
||||
continue
|
||||
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.multimodal.processing.processor import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.transformers_utils.configs.deepseek_vl2 import (
|
||||
DeepseekVLV2Config,
|
||||
MlpProjectorConfig,
|
||||
@@ -244,6 +245,7 @@ class DeepseekVL2MultiModalProcessor(
|
||||
) -> BatchFeature:
|
||||
if not mm_data:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(tokenizer, HfTokenizer)
|
||||
return tokenizer(prompt, add_special_tokens=True, return_tensors="pt")
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
@@ -291,6 +293,7 @@ class DeepseekVL2MultiModalProcessor(
|
||||
if isinstance(images, ImageEmbeddingItems):
|
||||
num_image_tokens = images.get_feature_size(item_idx)
|
||||
else:
|
||||
assert isinstance(images, ImageProcessorItems)
|
||||
image_size = images.get_image_size(item_idx)
|
||||
|
||||
num_image_tokens = self.info.get_num_image_tokens(
|
||||
@@ -406,7 +409,9 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
return parent, names[-1]
|
||||
|
||||
# patch for timm ViT instance to support tensor parallel
|
||||
def patch_vit_for_tp(self, vit: torch.nn.Module, quant_config: QuantizationConfig):
|
||||
def patch_vit_for_tp(
|
||||
self, vit: torch.nn.Module, quant_config: QuantizationConfig | None
|
||||
):
|
||||
try:
|
||||
import timm
|
||||
except ImportError as e:
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
)
|
||||
@@ -195,8 +196,10 @@ class DiffusionGemmaForConditionalGeneration(
|
||||
|
||||
# ---- Vision tower ----
|
||||
vision_config = getattr(config, "vision_config", None)
|
||||
self.embed_vision: Gemma4MultimodalEmbedder | None
|
||||
if vision_config is not None:
|
||||
quant_config = vllm_config.quant_config
|
||||
tower_quant: QuantizationConfig | None
|
||||
if quant_config and quant_config.get_name() in [
|
||||
"bitsandbytes",
|
||||
"torchao",
|
||||
@@ -1048,7 +1051,7 @@ class DiffusionSampler:
|
||||
sampler: Any,
|
||||
diffusion_config: Any,
|
||||
vocab_size: int,
|
||||
diffusion_states: DiffusionGemmaRequestStates | None = None,
|
||||
diffusion_states: DiffusionGemmaRequestStates,
|
||||
*,
|
||||
confidence_threshold: float,
|
||||
t_min: float,
|
||||
|
||||
@@ -149,7 +149,7 @@ class DotsOCRProcessingInfo(Qwen2VLProcessingInfo):
|
||||
self,
|
||||
**kwargs: object,
|
||||
) -> Qwen2VLProcessor:
|
||||
self.get_tokenizer().image_token = IMAGE_TOKEN # Ensure image token is set
|
||||
setattr(self.get_tokenizer(), "image_token", IMAGE_TOKEN) # noqa: B010
|
||||
processor = self.ctx.get_hf_processor(
|
||||
Qwen2VLProcessor,
|
||||
**kwargs,
|
||||
@@ -430,7 +430,7 @@ class DotsVisionBlock(nn.Module):
|
||||
*,
|
||||
cu_seqlens: torch.Tensor,
|
||||
rotary_pos_emb: torch.Tensor,
|
||||
max_seqlen: int | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = hidden_states + self.attn(
|
||||
self.norm1(hidden_states),
|
||||
@@ -483,6 +483,7 @@ class DotsVisionTransformer(nn.Module):
|
||||
)
|
||||
if require_post_norm is None:
|
||||
require_post_norm = len(self.blocks) == config.num_hidden_layers
|
||||
self.post_trunk_norm: RMSNorm | None
|
||||
if require_post_norm and self.config.post_norm:
|
||||
self.post_trunk_norm = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)
|
||||
else:
|
||||
@@ -536,7 +537,7 @@ class DotsVisionTransformer(nn.Module):
|
||||
rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
|
||||
return rotary_pos_emb
|
||||
|
||||
def compute_attn_mask_seqlen(self, cu_seqlens: torch.Tensor) -> int | None:
|
||||
def compute_attn_mask_seqlen(self, cu_seqlens: torch.Tensor) -> torch.Tensor | None:
|
||||
max_seqlen = None
|
||||
if self.attn_backend in {
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
@@ -552,15 +553,17 @@ class DotsVisionTransformer(nn.Module):
|
||||
rotary_pos_emb = self.rot_pos_emb(grid_thw)
|
||||
|
||||
# Convert grid_thw to tensor (always expecting list format now)
|
||||
grid_thw = torch.tensor(grid_thw, device=hidden_states.device, dtype=torch.long)
|
||||
grid_thw_tensor = torch.tensor(
|
||||
grid_thw, device=hidden_states.device, dtype=torch.long
|
||||
)
|
||||
hidden_states = hidden_states.to(self.dtype)
|
||||
hidden_states = self.patch_embed(hidden_states, grid_thw)
|
||||
hidden_states = self.patch_embed(hidden_states, grid_thw_tensor)
|
||||
|
||||
cu_seqlens = torch.repeat_interleave(
|
||||
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
|
||||
grid_thw_tensor[:, 1] * grid_thw_tensor[:, 2], grid_thw_tensor[:, 0]
|
||||
).cumsum(
|
||||
dim=0,
|
||||
dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32,
|
||||
dtype=grid_thw_tensor.dtype if torch.jit.is_tracing() else torch.int32,
|
||||
)
|
||||
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
|
||||
|
||||
@@ -620,6 +623,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("image"):
|
||||
return "<|img|><|imgpad|><|endofimg|>"
|
||||
return None
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
@@ -627,6 +631,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
|
||||
self.config: DotsOCRConfig = vllm_config.model_config.hf_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
assert multimodal_config is not None
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
if isinstance(self.config.vision_config, dict):
|
||||
vision_config = DotsVisionConfig(**self.config.vision_config)
|
||||
@@ -677,6 +682,8 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
|
||||
raise AssertionError
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: DotsOCRImageInputs
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
|
||||
@@ -118,8 +118,9 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
|
||||
width: int,
|
||||
height: int,
|
||||
num_images: int,
|
||||
overrides: ImageDummyOptions | None = None,
|
||||
overrides: BaseDummyOptions | None = None,
|
||||
) -> list[Image.Image]:
|
||||
assert overrides is None or isinstance(overrides, ImageDummyOptions)
|
||||
if num_images == 0:
|
||||
return []
|
||||
if overrides:
|
||||
|
||||
Reference in New Issue
Block a user