[Fusion] Support 3D activation quantization fusion and enable it for Pixtral (#59612)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
Misha Goin
parent
dd30786fef
commit
19c0bed584
@@ -32,6 +32,7 @@ from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn
|
||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -758,7 +759,7 @@ class FeedForward(nn.Module):
|
||||
|
||||
def forward(self, x):
|
||||
x, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(x)
|
||||
x = maybe_fused_act_quant(self.act_fn, x, self.down_proj)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
@@ -1259,7 +1260,7 @@ class PixtralHFMLP(nn.Module):
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_and_mul(gate_up)
|
||||
x = maybe_fused_act_quant(self.act_and_mul, gate_up, self.down_proj)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
Reference in New Issue
Block a user