[Fusion] Support 3D activation quantization fusion and enable it for Pixtral (#59612)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
Misha Goin
parent
dd30786fef
commit
19c0bed584
@@ -112,6 +112,9 @@ class FlashInferCuteDslNvFp4W4A16LinearKernel(NvFp4LinearKernel):
|
||||
class FlashInferCuteDslNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
"""NVFP4 GEMM via FlashInfer's cutedsl backend."""
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
return kNvfp4Dynamic
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
@@ -137,19 +140,26 @@ class FlashInferCuteDslNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
x: torch.Tensor | QuantizedActivation,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="flashinfer-cutedsl",
|
||||
)
|
||||
qa = as_quantized_activation(x, self.input_quant_key())
|
||||
if qa is not None:
|
||||
x_fp4, x_blockscale = qa.data, qa.scale
|
||||
x_fp4 = x_fp4.reshape(-1, x_fp4.shape[-1])
|
||||
output_dtype = qa.orig_dtype
|
||||
output_shape = [*qa.orig_shape[:-1], output_size]
|
||||
else:
|
||||
assert isinstance(x, torch.Tensor)
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
layer.input_global_scale_inv,
|
||||
is_sf_swizzled_layout=True,
|
||||
backend="flashinfer-cutedsl",
|
||||
)
|
||||
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4, nvfp4_weight_padding_bytes(layer)
|
||||
@@ -219,6 +229,7 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
qa = as_quantized_activation(x, self.input_quant_key())
|
||||
if qa is not None:
|
||||
x_fp4, x_blockscale = qa.data, qa.scale
|
||||
x_fp4 = x_fp4.reshape(-1, x_fp4.shape[-1])
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(x_fp4, weights_padding_bytes)
|
||||
output_dtype = qa.orig_dtype
|
||||
output_shape = [*qa.orig_shape[:-1], output_size]
|
||||
|
||||
@@ -63,10 +63,10 @@ def _silu_and_mul_fp8_dynamic_block(
|
||||
x: torch.Tensor, linear: LinearBase, group_size: int, quant_key: QuantKey
|
||||
) -> QuantizedActivation:
|
||||
"""SiluAndMul + FP8 dynamic per-block quantization."""
|
||||
assert x.ndim == 2, f"Input must be 2D [batch, hidden*2], got {x.shape}"
|
||||
|
||||
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
|
||||
d = x.shape[-1] // 2
|
||||
out_shape = x.shape[:-1] + (d,)
|
||||
x = x.reshape(-1, x.shape[-1])
|
||||
num_tokens = x.shape[0]
|
||||
num_groups = d // group_size
|
||||
|
||||
@@ -102,10 +102,10 @@ def _silu_and_mul_nvfp4_dynamic(
|
||||
x: torch.Tensor, linear: LinearBase
|
||||
) -> QuantizedActivation:
|
||||
"""SiluAndMul + NVFP4 dynamic quantization."""
|
||||
assert x.ndim == 2, f"Input must be 2D [batch, hidden*2], got {x.shape}"
|
||||
|
||||
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
|
||||
d = x.shape[-1] // 2
|
||||
out_shape = x.shape[:-1] + (d,)
|
||||
x = x.reshape(-1, x.shape[-1])
|
||||
num_tokens = x.shape[0]
|
||||
|
||||
# NVFP4 packs 2 values into 1 byte
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn
|
||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -758,7 +759,7 @@ class FeedForward(nn.Module):
|
||||
|
||||
def forward(self, x):
|
||||
x, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(x)
|
||||
x = maybe_fused_act_quant(self.act_fn, x, self.down_proj)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
@@ -1259,7 +1260,7 @@ class PixtralHFMLP(nn.Module):
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_and_mul(gate_up)
|
||||
x = maybe_fused_act_quant(self.act_and_mul, gate_up, self.down_proj)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
Reference in New Issue
Block a user