Merge remote-tracking branch 'upstream/main' into patches
This commit is contained in:
@@ -30,6 +30,9 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
|
||||
from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import (
|
||||
packed_qk_rope_,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.vision import (
|
||||
is_vit_use_data_parallel,
|
||||
@@ -37,6 +40,7 @@ from vllm.model_executor.models.vision import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -451,14 +455,19 @@ class MoonViTEncoderLayer(nn.Module):
|
||||
)
|
||||
# xqkv: (seqlen, 3, nheads, headdim)
|
||||
xqkv = xqkv.view(*qkv_shape)
|
||||
xq, xk, xv = torch.unbind(xqkv, dim=-3)
|
||||
|
||||
_apply_rope_input_validation(xq, rope_freqs_cis)
|
||||
_apply_rope_input_validation(xk, rope_freqs_cis)
|
||||
rope_cos = rope_freqs_cis.real.contiguous()
|
||||
rope_sin = rope_freqs_cis.imag.contiguous()
|
||||
xq = self.apply_rotary_emb(xq, rope_cos, rope_sin)
|
||||
xk = self.apply_rotary_emb(xk, rope_cos, rope_sin)
|
||||
# xq/xk alias xqkv, so the in-place rotation below is visible through
|
||||
# them; only the fallback needs to rebind.
|
||||
xq, xk, xv = torch.unbind(xqkv, dim=-3)
|
||||
if HAS_TRITON:
|
||||
packed_qk_rope_(xqkv, rope_freqs_cis)
|
||||
else:
|
||||
_apply_rope_input_validation(xq, rope_freqs_cis)
|
||||
_apply_rope_input_validation(xk, rope_freqs_cis)
|
||||
rope_cos = rope_freqs_cis.real.contiguous()
|
||||
rope_sin = rope_freqs_cis.imag.contiguous()
|
||||
xq = self.apply_rotary_emb(xq, rope_cos, rope_sin)
|
||||
xk = self.apply_rotary_emb(xk, rope_cos, rope_sin)
|
||||
|
||||
if max_seqlen is None:
|
||||
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
|
||||
|
||||
@@ -1055,15 +1055,18 @@ class OpenPanguVLForConditionalGeneration(
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.language_model.embed_input_ids(input_ids)
|
||||
if multimodal_embeddings is not None:
|
||||
inputs_embeds = self.embed_input_ids( # type: ignore[call-overload]
|
||||
input_ids,
|
||||
inputs_embeds,
|
||||
multimodal_embeddings,
|
||||
[self.config.image_token_id, self.config.video_token_id],
|
||||
)
|
||||
return inputs_embeds
|
||||
if multimodal_embeddings is None:
|
||||
return self.language_model.embed_input_ids(input_ids)
|
||||
|
||||
mm_token_ids = input_ids.new_tensor(
|
||||
[self.config.image_token_id, self.config.video_token_id]
|
||||
)
|
||||
is_multimodal = torch.isin(input_ids, mm_token_ids)
|
||||
return self.embed_input_ids(
|
||||
input_ids,
|
||||
multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: OpenPanguVLImageInputs
|
||||
|
||||
@@ -1297,7 +1297,7 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> None:
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ from vllm.model_executor.models.utils import (
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .bert_with_rope import BertWithRope, JinaRobertaModel
|
||||
from .interfaces import SupportsCrossEncoding
|
||||
from .interfaces import SupportsCrossEncoding, SupportsLoRA
|
||||
from .interfaces_base import attn_type, default_pooling_type
|
||||
|
||||
|
||||
@@ -266,7 +266,7 @@ class BgeM3EmbeddingModel(RobertaEmbeddingModel):
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="CLS")
|
||||
class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
|
||||
class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsLoRA):
|
||||
"""A model that uses Roberta to provide embedding functionalities.
|
||||
|
||||
This class encapsulates the BertModel and provides an interface for
|
||||
@@ -279,6 +279,7 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
|
||||
"""
|
||||
|
||||
is_pooling_model = True
|
||||
packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]}
|
||||
jina_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"emb_ln": "embeddings.LayerNorm",
|
||||
@@ -295,7 +296,8 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.padding_idx: int = vllm_config.model_config.hf_config.pad_token_id
|
||||
self.config = config
|
||||
self.padding_idx: int = config.pad_token_id
|
||||
|
||||
self.num_labels = config.num_labels
|
||||
self.roberta = BertModel(
|
||||
|
||||
@@ -282,10 +282,14 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
|
||||
|
||||
def get_replacement_ultravox(item_idx: int):
|
||||
if "audio_num_tokens" in out_mm_data:
|
||||
return [replacement_id] * int(out_mm_data["audio_num_tokens"][item_idx])
|
||||
audio_num_tokens = out_mm_data["audio_num_tokens"]
|
||||
assert isinstance(audio_num_tokens, torch.Tensor)
|
||||
return [replacement_id] * int(audio_num_tokens[item_idx])
|
||||
start = chunks_start_idx[item_idx]
|
||||
end = chunks_start_idx[item_idx + 1]
|
||||
audio_token_len = out_mm_data["audio_token_len"][start:end].sum()
|
||||
audio_token_lens = out_mm_data["audio_token_len"]
|
||||
assert isinstance(audio_token_lens, torch.Tensor)
|
||||
audio_token_len = audio_token_lens[start:end].sum()
|
||||
return [replacement_id] * int(audio_token_len) # type: ignore
|
||||
|
||||
return [
|
||||
@@ -566,7 +570,8 @@ class UltravoxWhisperEncoder(WhisperEncoder):
|
||||
) -> torch.Tensor:
|
||||
return (input_lengths - 1) // 2 + 1
|
||||
|
||||
def forward(
|
||||
# Ultravox requires chunk lengths in addition to Whisper's input features.
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_features: torch.Tensor,
|
||||
audio_lens: torch.Tensor,
|
||||
@@ -960,12 +965,12 @@ def pad_and_concat_to_dim3(
|
||||
|
||||
return features
|
||||
|
||||
features = [pad_and_concat_to_dim3(f) for f in features]
|
||||
tensors = [pad_and_concat_to_dim3(f) for f in features]
|
||||
|
||||
max_len = max(f.shape[-1] for f in features)
|
||||
max_len = max(f.shape[-1] for f in tensors)
|
||||
# Ensure all features have dim=3
|
||||
features = [f.view(-1, *f.shape[-2:]) for f in features]
|
||||
tensors = [f.view(-1, *f.shape[-2:]) for f in tensors]
|
||||
# Pad and concatenate:
|
||||
# [[B1, 80, M1], [B2, 80, M2]] -> [B1+B2, 80, max(M1, M2)]
|
||||
features = [F.pad(f, (0, max_len - f.shape[-1])) for f in features]
|
||||
return torch.cat(features)
|
||||
tensors = [F.pad(f, (0, max_len - f.shape[-1])) for f in tensors]
|
||||
return torch.cat(tensors)
|
||||
|
||||
@@ -199,6 +199,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
|
||||
if isinstance(images, ImageEmbeddingItems):
|
||||
num_image_tokens = images.get_feature_size(item_idx)
|
||||
else:
|
||||
assert isinstance(images, ImageProcessorItems)
|
||||
size = images.get_image_size(item_idx)
|
||||
|
||||
# Disable crop mode for multi-image input.
|
||||
|
||||
@@ -35,7 +35,6 @@ if TYPE_CHECKING:
|
||||
from transformers import PreTrainedConfig
|
||||
from transformers.conversion_mapping import WeightRenaming
|
||||
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -337,7 +336,7 @@ class AutoWeightsLoader:
|
||||
"""
|
||||
# Add persistent registered buffers.
|
||||
# Non-persistent buffers are excluded, matching PyTorch state_dict().
|
||||
non_persistent = getattr(module, "_non_persistent_buffers_set", set())
|
||||
non_persistent: set[str] = getattr(module, "_non_persistent_buffers_set", set())
|
||||
for buf_name, buf in module.named_buffers(recurse=False):
|
||||
if buf_name not in child_params and buf_name not in non_persistent:
|
||||
child_params[buf_name] = buf
|
||||
@@ -558,7 +557,7 @@ def maybe_fuse_shared_experts(
|
||||
|
||||
|
||||
def get_spec_layer_idx_from_weight_name(
|
||||
config: "ModelConfig", weight_name: str
|
||||
config: "PreTrainedConfig", weight_name: str
|
||||
) -> int | None:
|
||||
"""Return the MTP layer index a weight belongs to, or None.
|
||||
|
||||
@@ -587,7 +586,7 @@ def get_spec_layer_idx_from_weight_name(
|
||||
|
||||
|
||||
def skip_spec_layers(
|
||||
weights: Iterable[tuple[str, torch.Tensor]], config: "ModelConfig"
|
||||
weights: Iterable[tuple[str, torch.Tensor]], config: "PreTrainedConfig"
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Drop MTP spec-layer weights (loaded by the MTP head, not the base model).
|
||||
|
||||
@@ -629,11 +628,11 @@ def init_vllm_registered_model(
|
||||
|
||||
|
||||
@overload
|
||||
def flatten_bn(x: torch.Tensor) -> torch.Tensor: ...
|
||||
def flatten_bn(x: list[torch.Tensor]) -> list[torch.Tensor]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def flatten_bn(x: list[torch.Tensor]) -> list[torch.Tensor]: ...
|
||||
def flatten_bn(x: torch.Tensor) -> torch.Tensor: ...
|
||||
|
||||
|
||||
@overload
|
||||
@@ -973,7 +972,9 @@ def get_draft_quant_config(vllm_config: VllmConfig) -> "QuantizationConfig | Non
|
||||
The draft model's config if available, None otherwise.
|
||||
|
||||
"""
|
||||
draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
draft_load_config = vllm_config.load_config
|
||||
|
||||
return (
|
||||
|
||||
Reference in New Issue
Block a user