Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only PLaMo2 model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
@@ -22,31 +23,45 @@ from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator, MambaStateShapeCalculator)
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn, causal_conv1d_update)
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
|
||||
selective_state_update)
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_state_update
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen)
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
|
||||
DEFAULT_VOCAB_PADDING_SIZE,
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
composed_weight_loader, default_weight_loader, sharded_weight_loader)
|
||||
from vllm.model_executor.models.interfaces import (HasInnerState, IsHybrid,
|
||||
SupportsPP)
|
||||
composed_weight_loader,
|
||||
default_weight_loader,
|
||||
sharded_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import HasInnerState, IsHybrid, SupportsPP
|
||||
from vllm.model_executor.models.utils import (
|
||||
is_pp_missing_parameter, make_empty_intermediate_tensors_factory,
|
||||
make_layers, maybe_prefix)
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils import direct_register_custom_op
|
||||
@@ -89,12 +104,7 @@ def is_mamba(config: Plamo2Config, i: int) -> bool:
|
||||
# transformers.models.mamba.modeling_mamba.MambaMixer
|
||||
@CustomOp.register(name="plamo2_mamba_mixer")
|
||||
class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
|
||||
def __init__(self,
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
prefix: str = "",
|
||||
**kwargs) -> None:
|
||||
def __init__(self, vllm_config: VllmConfig, *, prefix: str = "", **kwargs) -> None:
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
@@ -103,8 +113,9 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
self.hidden_size = self.config.hidden_size
|
||||
self.ssm_state_size = self.config.mamba_d_state
|
||||
self.conv_kernel_size = self.config.mamba_d_conv
|
||||
self.intermediate_size = (self.config.mamba_num_heads *
|
||||
self.config.hidden_size_per_head)
|
||||
self.intermediate_size = (
|
||||
self.config.mamba_num_heads * self.config.hidden_size_per_head
|
||||
)
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.head_dim = self.config.hidden_size_per_head
|
||||
self.num_heads = self.config.mamba_num_heads
|
||||
@@ -155,17 +166,17 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
torch.empty(
|
||||
divide(self.num_heads, self.tp_size),
|
||||
dtype=torch.float32,
|
||||
))
|
||||
)
|
||||
)
|
||||
self.D = nn.Parameter(torch.ones(divide(self.num_heads, self.tp_size)))
|
||||
self.dt_bias = nn.Parameter(
|
||||
torch.ones(divide(self.num_heads, self.tp_size)))
|
||||
self.dt_bias = nn.Parameter(torch.ones(divide(self.num_heads, self.tp_size)))
|
||||
|
||||
set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)})
|
||||
a_weight_loader = composed_weight_loader(
|
||||
sharded_weight_loader(0), lambda x: -torch.exp(x.float()))
|
||||
sharded_weight_loader(0), lambda x: -torch.exp(x.float())
|
||||
)
|
||||
set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
|
||||
set_weight_attrs(self.dt_bias,
|
||||
{"weight_loader": sharded_weight_loader(0)})
|
||||
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
|
||||
|
||||
self.out_proj = RowParallelLinear(
|
||||
self.intermediate_size,
|
||||
@@ -179,12 +190,9 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
# The activation function is fixed to SiLU.
|
||||
self.activation = "silu"
|
||||
|
||||
self.dt_norm = RMSNorm(self.time_step_rank,
|
||||
eps=self.config.rms_norm_eps)
|
||||
self.B_norm = RMSNorm(self.ssm_state_size,
|
||||
eps=self.config.rms_norm_eps)
|
||||
self.C_norm = RMSNorm(self.ssm_state_size,
|
||||
eps=self.config.rms_norm_eps)
|
||||
self.dt_norm = RMSNorm(self.time_step_rank, eps=self.config.rms_norm_eps)
|
||||
self.B_norm = RMSNorm(self.ssm_state_size, eps=self.config.rms_norm_eps)
|
||||
self.C_norm = RMSNorm(self.ssm_state_size, eps=self.config.rms_norm_eps)
|
||||
|
||||
self.chunk_size = self.config.mamba_chunk_size
|
||||
|
||||
@@ -239,7 +247,6 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
output: torch.Tensor,
|
||||
**kwargs,
|
||||
):
|
||||
|
||||
forward_context = get_forward_context()
|
||||
# attn_metadata contains metadata necessary for the mamba2 triton
|
||||
# kernels to operate in continuous batching and in chunked prefill
|
||||
@@ -269,13 +276,15 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
gate, hidden_states = projected_states.chunk(2, dim=-1)
|
||||
|
||||
# 2. Convolution sequence transformation
|
||||
conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
|
||||
self.conv1d.weight.size(2))
|
||||
conv_weights = self.conv1d.weight.view(
|
||||
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
# profile run
|
||||
hidden_states = (hidden_states.transpose(0, 1).clone().transpose(
|
||||
0, 1)).contiguous()
|
||||
hidden_states = (
|
||||
hidden_states.transpose(0, 1).clone().transpose(0, 1)
|
||||
).contiguous()
|
||||
output[:] = self.out_proj(hidden_states)
|
||||
return
|
||||
|
||||
@@ -294,9 +303,9 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
[num_decodes, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
gate_d, gate_p = torch.split(gate[:num_actual_tokens],
|
||||
[num_decodes, num_prefill_tokens],
|
||||
dim=0)
|
||||
gate_d, gate_p = torch.split(
|
||||
gate[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
|
||||
)
|
||||
# Split along batch dimension
|
||||
state_indices_tensor_d, state_indices_tensor_p = torch.split(
|
||||
state_indices_tensor,
|
||||
@@ -309,7 +318,7 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
preallocated_ssm_out = torch.empty(
|
||||
[
|
||||
num_prefill_tokens + num_decodes,
|
||||
(self.num_heads // self.tp_size) * self.head_dim
|
||||
(self.num_heads // self.tp_size) * self.head_dim,
|
||||
],
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
@@ -325,8 +334,7 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
# 2. Convolution sequence transformation
|
||||
# - "cache_indices" updates the conv_state cache in positions
|
||||
# pointed to by "state_indices_tensor"
|
||||
x = hidden_states_p.transpose(
|
||||
0, 1) # this is the form that causal-conv see
|
||||
x = hidden_states_p.transpose(0, 1) # this is the form that causal-conv see
|
||||
hidden_states_p = causal_conv1d_fn(
|
||||
x,
|
||||
conv_weights,
|
||||
@@ -336,7 +344,8 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
has_initial_state=has_initial_states_p,
|
||||
cache_indices=state_indices_tensor_p,
|
||||
metadata=attn_metadata,
|
||||
query_start_loc=query_start_loc_p)
|
||||
query_start_loc=query_start_loc_p,
|
||||
)
|
||||
hidden_states_p = hidden_states_p.transpose(0, 1)
|
||||
hidden_states_p = hidden_states_p[:num_prefill_tokens]
|
||||
# In some instances, the following `bcdt_proj` op
|
||||
@@ -352,20 +361,23 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
# making a copy of the states
|
||||
initial_states = torch.where(
|
||||
has_initial_states_p[:, None, None, None],
|
||||
ssm_state[state_indices_tensor_p], 0)
|
||||
ssm_state[state_indices_tensor_p],
|
||||
0,
|
||||
)
|
||||
|
||||
varlen_state = mamba_chunk_scan_combined_varlen(
|
||||
hidden_states_p.view(num_prefill_tokens,
|
||||
self.num_heads // self.tp_size,
|
||||
self.head_dim),
|
||||
hidden_states_p.view(
|
||||
num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim
|
||||
),
|
||||
dt,
|
||||
self.A,
|
||||
B.view(num_prefill_tokens, 1, -1),
|
||||
C.view(num_prefill_tokens, 1, -1),
|
||||
chunk_size=chunk_size,
|
||||
D=self.D,
|
||||
z=gate_p.view(num_prefill_tokens,
|
||||
self.num_heads // self.tp_size, self.head_dim),
|
||||
z=gate_p.view(
|
||||
num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim
|
||||
),
|
||||
dt_bias=self.dt_bias,
|
||||
seq_idx=seq_idx_p,
|
||||
cu_seqlens=query_start_loc_p,
|
||||
@@ -374,8 +386,7 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
initial_states=initial_states,
|
||||
dt_softplus=True,
|
||||
dt_limit=(0.0, float("inf")),
|
||||
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1,
|
||||
self.head_dim),
|
||||
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
|
||||
state_dtype=ssm_state.dtype,
|
||||
)
|
||||
|
||||
@@ -392,21 +403,23 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
self.activation,
|
||||
conv_state_indices=state_indices_tensor_d)
|
||||
conv_state_indices=state_indices_tensor_d,
|
||||
)
|
||||
|
||||
B, C, dt = self._project_ssm_parameters(hidden_states_d)
|
||||
|
||||
# 3. State Space Model sequence transformation
|
||||
A = self.A[:, None, ...][:, :,
|
||||
None].expand(-1, self.head_dim,
|
||||
self.config.mamba_d_state)
|
||||
A = self.A[:, None, ...][:, :, None].expand(
|
||||
-1, self.head_dim, self.config.mamba_d_state
|
||||
)
|
||||
dt = dt[:, :, None].expand(-1, -1, self.head_dim)
|
||||
dt_bias = self.dt_bias[:, None, ...].expand(-1, self.head_dim)
|
||||
D = self.D[:, None, ...].expand(-1, self.head_dim)
|
||||
B = B.unsqueeze(1)
|
||||
C = C.unsqueeze(1)
|
||||
hidden_states_d = hidden_states_d.view(
|
||||
-1, self.num_heads // self.tp_size, self.head_dim)
|
||||
-1, self.num_heads // self.tp_size, self.head_dim
|
||||
)
|
||||
|
||||
# - the hidden is reshaped into (bs, num_heads, head_dim)
|
||||
# - ssm_state's slots will be selected
|
||||
@@ -425,8 +438,7 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_indices_tensor_d,
|
||||
out=preallocated_ssm_out_d.view(num_decodes, -1,
|
||||
self.head_dim),
|
||||
out=preallocated_ssm_out_d.view(num_decodes, -1, self.head_dim),
|
||||
)
|
||||
|
||||
# 4. Final linear projection
|
||||
@@ -457,8 +469,8 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
|
||||
return "mamba2"
|
||||
|
||||
def get_attn_backend(self) -> type["AttentionBackend"]:
|
||||
from vllm.v1.attention.backends.mamba2_attn import (
|
||||
Mamba2AttentionBackend)
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionBackend
|
||||
|
||||
return Mamba2AttentionBackend
|
||||
|
||||
|
||||
@@ -489,7 +501,6 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class DenseMLP(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Plamo2Config,
|
||||
@@ -508,12 +519,14 @@ class DenseMLP(nn.Module):
|
||||
return_bias=False,
|
||||
)
|
||||
self.act = SiluAndMul()
|
||||
self.down_proj = RowParallelLinear(self.intermediate_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
quant_config=quant_config,
|
||||
return_bias=False)
|
||||
self.down_proj = RowParallelLinear(
|
||||
self.intermediate_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
quant_config=quant_config,
|
||||
return_bias=False,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
h = self.gate_up_proj(hidden_states)
|
||||
@@ -522,12 +535,7 @@ class DenseMLP(nn.Module):
|
||||
|
||||
|
||||
class Plamo2AttentionMixer(nn.Module):
|
||||
|
||||
def __init__(self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
**kwargs) -> None:
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "", **kwargs) -> None:
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
@@ -560,20 +568,22 @@ class Plamo2AttentionMixer(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
self.rope_theta = config.rope_theta if hasattr(config,
|
||||
"rope_theta") else 10000
|
||||
self.rope_scaling = config.rope_scaling if hasattr(
|
||||
config, "rope_scaling") else None
|
||||
self.rope_theta = config.rope_theta if hasattr(config, "rope_theta") else 10000
|
||||
self.rope_scaling = (
|
||||
config.rope_scaling if hasattr(config, "rope_scaling") else None
|
||||
)
|
||||
max_position = config.max_position_embeddings
|
||||
if hasattr(vllm_config.model_config, "max_model_len") and isinstance(
|
||||
vllm_config.model_config.max_model_len, int):
|
||||
max_position = min(max_position,
|
||||
vllm_config.model_config.max_model_len)
|
||||
vllm_config.model_config.max_model_len, int
|
||||
):
|
||||
max_position = min(max_position, vllm_config.model_config.max_model_len)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
@@ -582,22 +592,24 @@ class Plamo2AttentionMixer(nn.Module):
|
||||
base=self.rope_theta,
|
||||
rope_scaling=self.rope_scaling,
|
||||
)
|
||||
self.q_norm = RMSNorm(config.hidden_size_per_head,
|
||||
eps=config.rms_norm_eps)
|
||||
self.q_norm = RMSNorm(config.hidden_size_per_head, eps=config.rms_norm_eps)
|
||||
self.q_norm.weight = torch.nn.Parameter(
|
||||
torch.ones((self.num_heads, config.hidden_size_per_head)))
|
||||
set_weight_attrs(self.q_norm.weight,
|
||||
{"weight_loader": sharded_weight_loader(0)})
|
||||
self.k_norm = RMSNorm(config.hidden_size_per_head,
|
||||
eps=config.rms_norm_eps)
|
||||
torch.ones((self.num_heads, config.hidden_size_per_head))
|
||||
)
|
||||
set_weight_attrs(
|
||||
self.q_norm.weight, {"weight_loader": sharded_weight_loader(0)}
|
||||
)
|
||||
self.k_norm = RMSNorm(config.hidden_size_per_head, eps=config.rms_norm_eps)
|
||||
self.k_norm.weight = torch.nn.Parameter(
|
||||
torch.ones((self.num_kv_heads, config.hidden_size_per_head)))
|
||||
torch.ones((self.num_kv_heads, config.hidden_size_per_head))
|
||||
)
|
||||
# Tensor-parallelism shards the K norm weights to the tp ranks
|
||||
# in a head-wise manner. This approach does not work if there is only
|
||||
# a single KV head, as is the case for PLaMo 2-1B.
|
||||
if self.total_num_kv_heads != 1:
|
||||
set_weight_attrs(self.k_norm.weight,
|
||||
{"weight_loader": sharded_weight_loader(0)})
|
||||
set_weight_attrs(
|
||||
self.k_norm.weight, {"weight_loader": sharded_weight_loader(0)}
|
||||
)
|
||||
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
@@ -631,35 +643,30 @@ class Plamo2AttentionMixer(nn.Module):
|
||||
|
||||
|
||||
class Plamo2DecoderLayer(nn.Module):
|
||||
|
||||
def __init__(self,
|
||||
vllm_config: VllmConfig,
|
||||
layer_idx: int,
|
||||
prefix: str = "",
|
||||
**kwargs) -> None:
|
||||
def __init__(
|
||||
self, vllm_config: VllmConfig, layer_idx: int, prefix: str = "", **kwargs
|
||||
) -> None:
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.is_mamba = is_mamba(config, layer_idx)
|
||||
if self.is_mamba:
|
||||
self.mixer = Plamo2MambaMixer(vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.mixer")
|
||||
self.mixer = Plamo2MambaMixer(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mixer"
|
||||
)
|
||||
else:
|
||||
self.mixer = Plamo2AttentionMixer(vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.mixer")
|
||||
self.mixer = Plamo2AttentionMixer(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.mixer"
|
||||
)
|
||||
|
||||
self.mlp = DenseMLP(config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp")
|
||||
self.pre_mixer_norm = RMSNorm(config.hidden_size,
|
||||
eps=config.rms_norm_eps)
|
||||
self.post_mixer_norm = RMSNorm(config.hidden_size,
|
||||
eps=config.rms_norm_eps)
|
||||
self.pre_mlp_norm = RMSNorm(config.hidden_size,
|
||||
eps=config.rms_norm_eps)
|
||||
self.post_mlp_norm = RMSNorm(config.hidden_size,
|
||||
eps=config.rms_norm_eps)
|
||||
self.mlp = DenseMLP(
|
||||
config=config, quant_config=quant_config, prefix=f"{prefix}.mlp"
|
||||
)
|
||||
self.pre_mixer_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_mixer_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.pre_mlp_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_mlp_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -672,8 +679,7 @@ class Plamo2DecoderLayer(nn.Module):
|
||||
residual = hidden_states
|
||||
hidden_states = self.pre_mixer_norm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.pre_mixer_norm(
|
||||
hidden_states, residual)
|
||||
hidden_states, residual = self.pre_mixer_norm(hidden_states, residual)
|
||||
|
||||
if self.is_mamba:
|
||||
# Plamo2MambaMixer writes output to this tensor
|
||||
@@ -700,7 +706,6 @@ class Plamo2DecoderLayer(nn.Module):
|
||||
|
||||
|
||||
class Plamo2Decoder(torch.nn.Module):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
@@ -708,13 +713,16 @@ class Plamo2Decoder(torch.nn.Module):
|
||||
|
||||
def get_layer(prefix: str):
|
||||
layer_idx = int(prefix.rsplit(".", 1)[1])
|
||||
return Plamo2DecoderLayer(vllm_config=vllm_config,
|
||||
layer_idx=layer_idx,
|
||||
prefix=prefix,
|
||||
**extra_kwargs)
|
||||
return Plamo2DecoderLayer(
|
||||
vllm_config=vllm_config,
|
||||
layer_idx=layer_idx,
|
||||
prefix=prefix,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers")
|
||||
config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -733,7 +741,6 @@ class Plamo2Decoder(torch.nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class Plamo2Model(torch.nn.Module):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -750,11 +757,10 @@ class Plamo2Model(torch.nn.Module):
|
||||
org_num_embeddings=config.vocab_size,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size))
|
||||
self.layers = Plamo2Decoder(vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.layers")
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
self.layers = Plamo2Decoder(vllm_config=vllm_config, prefix=f"{prefix}.layers")
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
@@ -784,10 +790,9 @@ class Plamo2Model(torch.nn.Module):
|
||||
residual=residual,
|
||||
)
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({
|
||||
"hidden_states": hidden_states,
|
||||
"residual": residual
|
||||
})
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
@@ -816,8 +821,9 @@ class Plamo2ForCausalLM(torch.nn.Module, HasInnerState, SupportsPP, IsHybrid):
|
||||
# the case for PLaMo2, as indicated by the FIXME comment.
|
||||
self.config.head_dim = self.config.hidden_size_per_head
|
||||
|
||||
self.model = Plamo2Model(vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"))
|
||||
self.model = Plamo2Model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.vocab_size = self.config.vocab_size
|
||||
self.unpadded_vocab_size = self.config.vocab_size
|
||||
num_embeddings = ((self.vocab_size + 15) // 16) * 16
|
||||
@@ -831,23 +837,27 @@ class Plamo2ForCausalLM(torch.nn.Module, HasInnerState, SupportsPP, IsHybrid):
|
||||
if self.config.tie_word_embeddings:
|
||||
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
|
||||
|
||||
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
|
||||
self.config.vocab_size)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.unpadded_vocab_size, self.config.vocab_size
|
||||
)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors)
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.get_input_embeddings(input_ids)
|
||||
|
||||
def forward(self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs):
|
||||
|
||||
hidden_states = self.model(input_ids, positions, intermediate_tensors,
|
||||
inputs_embeds)
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs,
|
||||
):
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
@classmethod
|
||||
@@ -855,7 +865,6 @@ class Plamo2ForCausalLM(torch.nn.Module, HasInnerState, SupportsPP, IsHybrid):
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
|
||||
return MambaStateDtypeCalculator.mamba2_state_dtype(
|
||||
vllm_config.model_config.dtype,
|
||||
vllm_config.cache_config.mamba_cache_dtype,
|
||||
@@ -877,8 +886,7 @@ class Plamo2ForCausalLM(torch.nn.Module, HasInnerState, SupportsPP, IsHybrid):
|
||||
"""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
intermediate_size =\
|
||||
hf_config.mamba_num_heads * hf_config.hidden_size_per_head
|
||||
intermediate_size = hf_config.mamba_num_heads * hf_config.hidden_size_per_head
|
||||
|
||||
return MambaStateShapeCalculator.mamba2_state_shape(
|
||||
intermediate_size=intermediate_size,
|
||||
@@ -900,7 +908,6 @@ class Plamo2ForCausalLM(torch.nn.Module, HasInnerState, SupportsPP, IsHybrid):
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
params_dict = dict(self.named_parameters())
|
||||
for name, loaded_weight in weights:
|
||||
|
||||
# Both tie_word_embeddings=True and lm_head.weight in the safetensor
|
||||
# at the same time causes dict key access error.
|
||||
if name == "lm_head.weight" and self.config.tie_word_embeddings:
|
||||
@@ -932,10 +939,12 @@ class Plamo2ForCausalLM(torch.nn.Module, HasInnerState, SupportsPP, IsHybrid):
|
||||
# Also, in addition to the quantized weights,
|
||||
# the zero points and scales have to be reshaped as well.
|
||||
# Packing should not be affected by this.
|
||||
if ".mixer.in_proj.weight" in name \
|
||||
or "mixer.in_proj.qweight" in name \
|
||||
or "mixer.in_proj.scales" in name \
|
||||
or "mixer.in_proj.qzeros" in name:
|
||||
if (
|
||||
".mixer.in_proj.weight" in name
|
||||
or "mixer.in_proj.qweight" in name
|
||||
or "mixer.in_proj.scales" in name
|
||||
or "mixer.in_proj.qzeros" in name
|
||||
):
|
||||
if "mixer.in_proj.weight" in name:
|
||||
loaded_weight = loaded_weight.transpose(0, 1)
|
||||
# for weight:
|
||||
@@ -945,14 +954,14 @@ class Plamo2ForCausalLM(torch.nn.Module, HasInnerState, SupportsPP, IsHybrid):
|
||||
# for scales and qzeros:
|
||||
# loaded_weight.shape[0] == self.config.hidden_size // self.vllm_config.quant_config.group_size # noqa
|
||||
loaded_weight = loaded_weight.reshape(
|
||||
loaded_weight.shape[0], self.config.mamba_num_heads, -1)
|
||||
gate_weight, hidden_states_weight = loaded_weight.chunk(2,
|
||||
dim=-1)
|
||||
loaded_weight.shape[0], self.config.mamba_num_heads, -1
|
||||
)
|
||||
gate_weight, hidden_states_weight = loaded_weight.chunk(2, dim=-1)
|
||||
gate_weight = gate_weight.reshape(loaded_weight.shape[0], -1)
|
||||
hidden_states_weight = hidden_states_weight.reshape(
|
||||
loaded_weight.shape[0], -1)
|
||||
loaded_weight = torch.cat([gate_weight, hidden_states_weight],
|
||||
dim=-1)
|
||||
loaded_weight.shape[0], -1
|
||||
)
|
||||
loaded_weight = torch.cat([gate_weight, hidden_states_weight], dim=-1)
|
||||
if "mixer.in_proj.weight" in name:
|
||||
loaded_weight = loaded_weight.transpose(0, 1)
|
||||
|
||||
@@ -973,6 +982,5 @@ class Plamo2ForCausalLM(torch.nn.Module, HasInnerState, SupportsPP, IsHybrid):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader",
|
||||
default_weight_loader)
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
Reference in New Issue
Block a user