[Fusion] Support 3D activation quantization fusion and enable it for Pixtral (#59612)

Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Jakub Zakrzewski
2026-10-09 15:12:03 +08:00
committed by GitHub
co-authored by Codex Misha Goin
parent dd30786fef
commit 19c0bed584
6 changed files with 101 additions and 29 deletions
+19 -13
View File
@@ -115,13 +115,13 @@ class MockLinearRequiresUnquantized(MockLinearFp8Static):
requires_unquantized_input = True
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
@pytest.mark.parametrize("token_shape", [(1,), (16,), (128,), (1, 17), (2, 17)])
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode()
def test_maybe_fused_act_quant_fp8_static(
default_vllm_config,
num_tokens: int,
token_shape: tuple[int, ...],
hidden_size: int,
dtype: torch.dtype,
) -> None:
@@ -133,26 +133,29 @@ def test_maybe_fused_act_quant_fp8_static(
scale = torch.tensor([0.5], device=device, dtype=torch.float32)
linear = MockLinearFp8Static(scale)
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device)
x = torch.randn(*token_shape, hidden_size * 2, dtype=dtype, device=device)
result = maybe_fused_act_quant(act_fn, x, linear)
assert isinstance(result, QuantizedActivation)
assert result.quant_key == kFp8StaticTensorSym
assert result.data.dtype == current_platform.fp8_dtype()
assert result.orig_dtype == dtype
assert result.orig_shape == (num_tokens, hidden_size)
assert result.orig_shape == (*token_shape, hidden_size)
assert result.data.shape == result.orig_shape
ref_out = ref_impl(act_fn, x, scale)
ref_out = ref_impl(act_fn, x.reshape(-1, hidden_size * 2), scale).reshape(
result.orig_shape
)
torch.testing.assert_close(result.data.to(torch.float32), ref_out.to(torch.float32))
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
@pytest.mark.parametrize("token_shape", [(1,), (16,), (128,), (1, 17), (2, 17)])
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode()
def test_maybe_fused_act_quant_fp8_dynamic_block(
default_vllm_config,
num_tokens: int,
token_shape: tuple[int, ...],
hidden_size: int,
dtype: torch.dtype,
) -> None:
@@ -166,31 +169,34 @@ def test_maybe_fused_act_quant_fp8_dynamic_block(
linear = MockLinearFp8Dynamic128()
scale = 1 / hidden_size
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device) * scale
x = torch.randn(*token_shape, hidden_size * 2, dtype=dtype, device=device) * scale
result = maybe_fused_act_quant(act_fn, x, linear)
assert isinstance(result, QuantizedActivation)
assert result.quant_key == kFp8Dynamic128Sym
assert result.data.dtype == current_platform.fp8_dtype()
assert result.orig_dtype == dtype
assert result.orig_shape == (num_tokens, hidden_size)
assert result.orig_shape == (*token_shape, hidden_size)
assert result.data.shape == result.orig_shape
num_groups = hidden_size // group_size
assert result.scale.shape == (num_tokens, num_groups)
assert result.scale.shape == (*token_shape, num_groups)
gate, up = x.split(hidden_size, dim=-1)
silu_out = F.silu(gate) * up
ref_out, ref_scales = per_token_group_quant_fp8(
silu_out, group_size=group_size, use_ue8m0=False
silu_out.reshape(-1, hidden_size), group_size=group_size, use_ue8m0=False
)
ref_out = ref_out.reshape(result.orig_shape)
ref_scales = ref_scales.reshape(*token_shape, num_groups)
torch.testing.assert_close(result.scale, ref_scales, rtol=1e-5, atol=1e-5)
ref_deq = ref_out.to(torch.float32) * ref_scales.repeat_interleave(
group_size, dim=1
group_size, dim=-1
)
result_deq = result.data.to(torch.float32) * result.scale.repeat_interleave(
group_size, dim=1
group_size, dim=-1
)
torch.testing.assert_close(ref_deq, result_deq, atol=5e-2, rtol=5e-2)