[Kernel] Support MulAndSilu (#11624)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -87,6 +87,41 @@ class SiluAndMul(CustomOp):
|
||||
return out
|
||||
|
||||
|
||||
@CustomOp.register("mul_and_silu")
|
||||
class MulAndSilu(CustomOp):
|
||||
"""An activation function for SwiGLU.
|
||||
|
||||
The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2.
|
||||
|
||||
Shapes:
|
||||
x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
|
||||
return: (num_tokens, d) or (batch_size, seq_len, d)
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
if current_platform.is_cuda_alike() or current_platform.is_cpu():
|
||||
self.op = torch.ops._C.mul_and_silu
|
||||
elif current_platform.is_xpu():
|
||||
from vllm._ipex_ops import ipex_ops
|
||||
self.op = ipex_ops.silu_and_mul
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""PyTorch-native implementation equivalent to forward()."""
|
||||
d = x.shape[-1] // 2
|
||||
return x[..., :d] * F.silu(x[..., d:])
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
output_shape = (x.shape[:-1] + (d, ))
|
||||
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
|
||||
self.op(out, x)
|
||||
return out
|
||||
|
||||
# TODO implement forward_xpu for MulAndSilu
|
||||
# def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
|
||||
@CustomOp.register("gelu_and_mul")
|
||||
class GeluAndMul(CustomOp):
|
||||
"""An activation function for GeGLU.
|
||||
|
||||
@@ -23,7 +23,8 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
|
||||
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
|
||||
InputContext, token_inputs)
|
||||
from vllm.model_executor import SamplingMetadata
|
||||
from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul
|
||||
from vllm.model_executor.layers.activation import (MulAndSilu, QuickGELU,
|
||||
SiluAndMul)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
@@ -462,15 +463,6 @@ class MolmoAttention(nn.Module):
|
||||
return output
|
||||
|
||||
|
||||
class SwiGLU(nn.Module):
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x, gate = x.chunk(2, dim=-1)
|
||||
# Note that the order is reversed compared to
|
||||
# SiluAndMul.
|
||||
return x * F.silu(gate)
|
||||
|
||||
|
||||
class LanuageModelMLP(nn.Module):
|
||||
"""Molmo's LLM mlp."""
|
||||
|
||||
@@ -489,7 +481,7 @@ class LanuageModelMLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
)
|
||||
# Activation function.
|
||||
self.act_fn = SwiGLU()
|
||||
self.act_fn = MulAndSilu()
|
||||
# Feed-forward output projection.
|
||||
self.down_proj = RowParallelLinear(
|
||||
self.intermediate_size,
|
||||
|
||||
@@ -16,7 +16,7 @@ from transformers.models.whisper.modeling_whisper import WhisperEncoder
|
||||
from vllm import envs
|
||||
from vllm.attention import AttentionMetadata
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
|
||||
from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
|
||||
from vllm.model_executor.model_loader.loader import DefaultModelLoader
|
||||
@@ -248,15 +248,6 @@ class StackAudioFrames(nn.Module):
|
||||
return audio_embeds
|
||||
|
||||
|
||||
class FlippedSiluAndMul(SiluAndMul):
|
||||
"""Ultravox is trained with SwiGLU with flipped halves."""
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
a, b = x.chunk(2, dim=-1)
|
||||
flipped = torch.cat((b, a), dim=-1)
|
||||
return super().forward(flipped)
|
||||
|
||||
|
||||
class UltravoxProjector(nn.Module):
|
||||
|
||||
def __init__(self, config: UltravoxConfig):
|
||||
@@ -269,7 +260,7 @@ class UltravoxProjector(nn.Module):
|
||||
dim = self.hidden_dim
|
||||
|
||||
if config.projector_act == "swiglu":
|
||||
self.act = FlippedSiluAndMul()
|
||||
self.act = MulAndSilu()
|
||||
dim = dim // 2
|
||||
else:
|
||||
self.act = get_act_fn(config.projector_act)
|
||||
|
||||
Reference in New Issue
Block a user