[Kernel] Support MulAndSilu (#11624)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -23,7 +23,8 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
|
||||
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
|
||||
InputContext, token_inputs)
|
||||
from vllm.model_executor import SamplingMetadata
|
||||
from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul
|
||||
from vllm.model_executor.layers.activation import (MulAndSilu, QuickGELU,
|
||||
SiluAndMul)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
@@ -462,15 +463,6 @@ class MolmoAttention(nn.Module):
|
||||
return output
|
||||
|
||||
|
||||
class SwiGLU(nn.Module):
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x, gate = x.chunk(2, dim=-1)
|
||||
# Note that the order is reversed compared to
|
||||
# SiluAndMul.
|
||||
return x * F.silu(gate)
|
||||
|
||||
|
||||
class LanuageModelMLP(nn.Module):
|
||||
"""Molmo's LLM mlp."""
|
||||
|
||||
@@ -489,7 +481,7 @@ class LanuageModelMLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
)
|
||||
# Activation function.
|
||||
self.act_fn = SwiGLU()
|
||||
self.act_fn = MulAndSilu()
|
||||
# Feed-forward output projection.
|
||||
self.down_proj = RowParallelLinear(
|
||||
self.intermediate_size,
|
||||
|
||||
@@ -16,7 +16,7 @@ from transformers.models.whisper.modeling_whisper import WhisperEncoder
|
||||
from vllm import envs
|
||||
from vllm.attention import AttentionMetadata
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
|
||||
from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
|
||||
from vllm.model_executor.model_loader.loader import DefaultModelLoader
|
||||
@@ -248,15 +248,6 @@ class StackAudioFrames(nn.Module):
|
||||
return audio_embeds
|
||||
|
||||
|
||||
class FlippedSiluAndMul(SiluAndMul):
|
||||
"""Ultravox is trained with SwiGLU with flipped halves."""
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
a, b = x.chunk(2, dim=-1)
|
||||
flipped = torch.cat((b, a), dim=-1)
|
||||
return super().forward(flipped)
|
||||
|
||||
|
||||
class UltravoxProjector(nn.Module):
|
||||
|
||||
def __init__(self, config: UltravoxConfig):
|
||||
@@ -269,7 +260,7 @@ class UltravoxProjector(nn.Module):
|
||||
dim = self.hidden_dim
|
||||
|
||||
if config.projector_act == "swiglu":
|
||||
self.act = FlippedSiluAndMul()
|
||||
self.act = MulAndSilu()
|
||||
dim = dim // 2
|
||||
else:
|
||||
self.act = get_act_fn(config.projector_act)
|
||||
|
||||
Reference in New Issue
Block a user