Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -10,7 +10,9 @@ import torch.nn as nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead, VocabParallelEmbedding)
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from .utils import maybe_prefix
|
||||
@@ -74,8 +76,7 @@ class MLPSpeculator(nn.Module):
|
||||
self.n_predict = config.n_predict
|
||||
self.vocab_size = config.vocab_size
|
||||
self.emb_dim = config.emb_dim
|
||||
self.inner_dim = config.inner_dim if config.inner_dim != 0 \
|
||||
else config.emb_dim
|
||||
self.inner_dim = config.inner_dim if config.inner_dim != 0 else config.emb_dim
|
||||
|
||||
self.max_speculative_tokens = config.num_lookahead_tokens
|
||||
|
||||
@@ -83,72 +84,93 @@ class MLPSpeculator(nn.Module):
|
||||
self.scale_input = config.scale_input
|
||||
|
||||
if self.tie_weights:
|
||||
assert (
|
||||
self.n_predict > 1
|
||||
), "You cannot tie weights between stages when only 1 exists"
|
||||
assert self.n_predict > 1, (
|
||||
"You cannot tie weights between stages when only 1 exists"
|
||||
)
|
||||
embedding = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
self.inner_dim,
|
||||
org_num_embeddings=config.vocab_size)
|
||||
config.vocab_size, self.inner_dim, org_num_embeddings=config.vocab_size
|
||||
)
|
||||
self.emb = nn.ModuleList([embedding] * self.max_speculative_tokens)
|
||||
|
||||
# the initial projection from the base model may
|
||||
# have a different size, so that stays separate.
|
||||
proj_first = nn.Linear(self.emb_dim, self.inner_dim, bias=False)
|
||||
proj_tied = nn.Linear(self.inner_dim, self.inner_dim, bias=False)
|
||||
self.proj = nn.ModuleList([proj_first] + [proj_tied] *
|
||||
(self.max_speculative_tokens - 1))
|
||||
self.proj = nn.ModuleList(
|
||||
[proj_first] + [proj_tied] * (self.max_speculative_tokens - 1)
|
||||
)
|
||||
|
||||
self.head = nn.ModuleList([
|
||||
ParallelLMHead(self.vocab_size,
|
||||
self.inner_dim,
|
||||
bias=False,
|
||||
prefix=maybe_prefix(prefix, f"head.{i}"))
|
||||
for i in range(self.max_speculative_tokens)
|
||||
])
|
||||
self.head = nn.ModuleList(
|
||||
[
|
||||
ParallelLMHead(
|
||||
self.vocab_size,
|
||||
self.inner_dim,
|
||||
bias=False,
|
||||
prefix=maybe_prefix(prefix, f"head.{i}"),
|
||||
)
|
||||
for i in range(self.max_speculative_tokens)
|
||||
]
|
||||
)
|
||||
|
||||
ln = MLPSpeculatorLayerNorm(self.inner_dim,
|
||||
elementwise_scale_and_shift=True)
|
||||
ln = MLPSpeculatorLayerNorm(
|
||||
self.inner_dim, elementwise_scale_and_shift=True
|
||||
)
|
||||
self.ln = nn.ModuleList([ln] * self.max_speculative_tokens)
|
||||
|
||||
else:
|
||||
self.emb = nn.ModuleList([
|
||||
VocabParallelEmbedding(config.vocab_size,
|
||||
self.inner_dim,
|
||||
org_num_embeddings=config.vocab_size)
|
||||
for _ in range(self.max_speculative_tokens)
|
||||
])
|
||||
self.emb = nn.ModuleList(
|
||||
[
|
||||
VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
self.inner_dim,
|
||||
org_num_embeddings=config.vocab_size,
|
||||
)
|
||||
for _ in range(self.max_speculative_tokens)
|
||||
]
|
||||
)
|
||||
|
||||
self.proj = nn.ModuleList([
|
||||
nn.Linear((self.emb_dim if i == 0 else self.inner_dim),
|
||||
self.inner_dim,
|
||||
bias=False)
|
||||
for i in range(self.max_speculative_tokens)
|
||||
])
|
||||
self.proj = nn.ModuleList(
|
||||
[
|
||||
nn.Linear(
|
||||
(self.emb_dim if i == 0 else self.inner_dim),
|
||||
self.inner_dim,
|
||||
bias=False,
|
||||
)
|
||||
for i in range(self.max_speculative_tokens)
|
||||
]
|
||||
)
|
||||
|
||||
self.head = nn.ModuleList([
|
||||
ParallelLMHead(self.vocab_size,
|
||||
self.inner_dim,
|
||||
bias=False,
|
||||
prefix=maybe_prefix(prefix, f"head.{i}"))
|
||||
for i in range(self.max_speculative_tokens)
|
||||
])
|
||||
self.ln = nn.ModuleList([
|
||||
MLPSpeculatorLayerNorm(self.inner_dim,
|
||||
elementwise_scale_and_shift=True)
|
||||
for _ in range(self.max_speculative_tokens)
|
||||
])
|
||||
self.head = nn.ModuleList(
|
||||
[
|
||||
ParallelLMHead(
|
||||
self.vocab_size,
|
||||
self.inner_dim,
|
||||
bias=False,
|
||||
prefix=maybe_prefix(prefix, f"head.{i}"),
|
||||
)
|
||||
for i in range(self.max_speculative_tokens)
|
||||
]
|
||||
)
|
||||
self.ln = nn.ModuleList(
|
||||
[
|
||||
MLPSpeculatorLayerNorm(
|
||||
self.inner_dim, elementwise_scale_and_shift=True
|
||||
)
|
||||
for _ in range(self.max_speculative_tokens)
|
||||
]
|
||||
)
|
||||
if self.scale_input:
|
||||
self.ln0 = MLPSpeculatorLayerNorm(
|
||||
self.emb_dim, elementwise_scale_and_shift=False)
|
||||
self.emb_dim, elementwise_scale_and_shift=False
|
||||
)
|
||||
|
||||
self.state_weight = 0.5**(0.5 / config.n_predict)
|
||||
self.emb_weight = math.sqrt(
|
||||
(1 - self.state_weight**2) * (self.inner_dim / 2))
|
||||
self.state_weight = 0.5 ** (0.5 / config.n_predict)
|
||||
self.emb_weight = math.sqrt((1 - self.state_weight**2) * (self.inner_dim / 2))
|
||||
self.activation = nn.GELU()
|
||||
self.config = config
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size,
|
||||
config.vocab_size, 1.0)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
config.vocab_size, config.vocab_size, 1.0
|
||||
)
|
||||
|
||||
# NOTE(woosuk): This method is commented out because it is old code
|
||||
# using V0. We should either port it to V1 or remove it.
|
||||
@@ -201,16 +223,14 @@ class MLPSpeculator(nn.Module):
|
||||
|
||||
# return next_tokens
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str,
|
||||
torch.Tensor]]) -> set[str]:
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
name = name.replace("speculator.", "")
|
||||
param = params_dict.get(name)
|
||||
if param is not None:
|
||||
weight_loader = getattr(param, "weight_loader",
|
||||
default_weight_loader)
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
Reference in New Issue
Block a user