[Fusion] Support 3D activation quantization fusion and enable it for Pixtral (#59612)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
Misha Goin
parent
dd30786fef
commit
19c0bed584
@@ -63,10 +63,10 @@ def _silu_and_mul_fp8_dynamic_block(
|
||||
x: torch.Tensor, linear: LinearBase, group_size: int, quant_key: QuantKey
|
||||
) -> QuantizedActivation:
|
||||
"""SiluAndMul + FP8 dynamic per-block quantization."""
|
||||
assert x.ndim == 2, f"Input must be 2D [batch, hidden*2], got {x.shape}"
|
||||
|
||||
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
|
||||
d = x.shape[-1] // 2
|
||||
out_shape = x.shape[:-1] + (d,)
|
||||
x = x.reshape(-1, x.shape[-1])
|
||||
num_tokens = x.shape[0]
|
||||
num_groups = d // group_size
|
||||
|
||||
@@ -102,10 +102,10 @@ def _silu_and_mul_nvfp4_dynamic(
|
||||
x: torch.Tensor, linear: LinearBase
|
||||
) -> QuantizedActivation:
|
||||
"""SiluAndMul + NVFP4 dynamic quantization."""
|
||||
assert x.ndim == 2, f"Input must be 2D [batch, hidden*2], got {x.shape}"
|
||||
|
||||
assert x.ndim in (2, 3), f"Input must be 2D or 3D, got {x.shape}"
|
||||
d = x.shape[-1] // 2
|
||||
out_shape = x.shape[:-1] + (d,)
|
||||
x = x.reshape(-1, x.shape[-1])
|
||||
num_tokens = x.shape[0]
|
||||
|
||||
# NVFP4 packs 2 values into 1 byte
|
||||
|
||||
Reference in New Issue
Block a user