[Refactor] Remove dead code multiple places (#58002)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -114,34 +114,6 @@ class InternVisionEmbeddings(nn.Module):
|
||||
return embeddings
|
||||
|
||||
|
||||
class InternVisionPatchModel(nn.Module):
|
||||
def __init__(self, config: PretrainedConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.embeddings = InternVisionEmbeddings(config)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.embeddings
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.Tensor | None = None,
|
||||
pixel_embeds: torch.Tensor | None = None,
|
||||
) -> torch.FloatTensor:
|
||||
if pixel_values is None and pixel_embeds is None:
|
||||
raise ValueError("You have to specify pixel_values or pixel_embeds")
|
||||
|
||||
if pixel_embeds is not None:
|
||||
hidden_states = pixel_embeds
|
||||
elif pixel_values is not None:
|
||||
if pixel_values.ndim == 4:
|
||||
hidden_states = self.embeddings(pixel_values)
|
||||
else:
|
||||
raise ValueError(f"wrong pixel_values size: {pixel_values.shape}")
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class InternParallelAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper."""
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
# limitations under the License.
|
||||
import math
|
||||
from collections.abc import Hashable, Iterable, Mapping
|
||||
from itertools import tee
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
import torch
|
||||
@@ -1041,25 +1040,6 @@ class Llama4ForConditionalGeneration(
|
||||
) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def separate_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
prefix: str,
|
||||
) -> tuple[Iterable[tuple[str, torch.Tensor]], Iterable[tuple[str, torch.Tensor]]]:
|
||||
weights1, weights2 = tee(weights, 2)
|
||||
|
||||
def get_prefix_weights() -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, data in weights1:
|
||||
if name.startswith(prefix):
|
||||
yield (name, data)
|
||||
|
||||
def get_other_weights() -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, data in weights2:
|
||||
if not name.startswith(prefix):
|
||||
yield (name, data)
|
||||
|
||||
return get_prefix_weights(), get_other_weights()
|
||||
|
||||
def _consolidate_qkv_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
|
||||
@@ -23,47 +23,14 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.models.llama import (
|
||||
LlamaDecoderLayer,
|
||||
LlamaForCausalLM,
|
||||
LlamaModel,
|
||||
)
|
||||
|
||||
|
||||
class TeleFLMModel(LlamaModel):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
layer_type: type[nn.Module] = LlamaDecoderLayer,
|
||||
):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix, layer_type=layer_type)
|
||||
"""
|
||||
This implementation is based on the µScaling paper presented at
|
||||
the ICLR 2025 Workshop:
|
||||
NanoLM: An Affordable LLM Study Benchmark \
|
||||
via Accurate Loss Prediction across Scales
|
||||
by Yiqun Yao et al.
|
||||
Available at: https://openreview.net/forum?id=IwaPYg1SCA
|
||||
arXiv preprint: https://arxiv.org/abs/2304.06875
|
||||
"""
|
||||
self.use_mup = self.config.use_mup
|
||||
if self.use_mup:
|
||||
self.input_mult = self.config.input_mult
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
embedding = self.embed_tokens(input_ids)
|
||||
if self.use_mup:
|
||||
embedding = embedding * self.input_mult
|
||||
return embedding
|
||||
|
||||
|
||||
class TeleFLMForCausalLM(LlamaForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
Reference in New Issue
Block a user