[Fusion] Manual ActivationQuantFusionPass initial application (#51415)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: ElizaWszola <ewszola@redhat.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: ElizaWszola <ewszola@redhat.com>
This commit is contained in:
Misha Goin
2026-09-03 13:37:31 -04:00
committed by GitHub
co-authored by Claude Opus 4.8 ElizaWszola
parent d4d703caf9
commit b7624069ff
10 changed files with 740 additions and 19 deletions
@@ -2,10 +2,20 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
import vllm._custom_ops as ops
from tests.kernels.utils import opcheck
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -69,3 +79,140 @@ def test_silu_and_mul(
ref_out.to(dtype=torch.float32), ops_out.to(dtype=torch.float32)
)
opcheck(torch.ops._C.silu_and_mul_quant, (ops_out, x, scale))
# ---------------------------------------------------------------------------
# Tests for maybe_fused_act_quant interface
# ---------------------------------------------------------------------------
class MockLinearFp8Static(torch.nn.Module):
"""Mock linear layer advertising kFp8StaticTensorSym."""
def __init__(self, input_scale: torch.Tensor):
super().__init__()
self.input_quant_key = kFp8StaticTensorSym
self.input_scale = input_scale
class MockLinearFp8Dynamic128(torch.nn.Module):
"""Mock linear layer advertising kFp8Dynamic128Sym."""
def __init__(self):
super().__init__()
self.input_quant_key = kFp8Dynamic128Sym
class MockLinearNoQuant(torch.nn.Module):
"""Mock linear layer with no input_quant_key (no fusion)."""
pass
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode()
def test_maybe_fused_act_quant_fp8_static(
default_vllm_config,
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
) -> None:
"""Test maybe_fused_act_quant with FP8 static per-tensor quantization."""
device = "cuda:0"
torch.set_default_device(device)
act_fn = SiluAndMul()
scale = torch.tensor([0.5], device=device, dtype=torch.float32)
linear = MockLinearFp8Static(scale)
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device)
result = maybe_fused_act_quant(act_fn, x, linear)
assert isinstance(result, QuantizedActivation)
assert result.quant_key == kFp8StaticTensorSym
assert result.data.dtype == current_platform.fp8_dtype()
assert result.orig_dtype == dtype
assert result.orig_shape == (num_tokens, hidden_size)
ref_out = ref_impl(act_fn, x, scale)
torch.testing.assert_close(result.data.to(torch.float32), ref_out.to(torch.float32))
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode()
def test_maybe_fused_act_quant_fp8_dynamic_block(
default_vllm_config,
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
) -> None:
"""Test maybe_fused_act_quant with FP8 dynamic per-block quantization."""
group_size = 128 # We only support 128 for now
device = "cuda:0"
torch.set_default_device(device)
act_fn = SiluAndMul()
linear = MockLinearFp8Dynamic128()
scale = 1 / hidden_size
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device) * scale
result = maybe_fused_act_quant(act_fn, x, linear)
assert isinstance(result, QuantizedActivation)
assert result.quant_key == kFp8Dynamic128Sym
assert result.data.dtype == current_platform.fp8_dtype()
assert result.orig_dtype == dtype
assert result.orig_shape == (num_tokens, hidden_size)
num_groups = hidden_size // group_size
assert result.scale.shape == (num_tokens, num_groups)
gate, up = x.split(hidden_size, dim=-1)
silu_out = F.silu(gate) * up
ref_out, ref_scales = per_token_group_quant_fp8(
silu_out, group_size=group_size, use_ue8m0=False
)
torch.testing.assert_close(result.scale, ref_scales, rtol=1e-5, atol=1e-5)
ref_deq = ref_out.to(torch.float32) * ref_scales.repeat_interleave(
group_size, dim=1
)
result_deq = result.data.to(torch.float32) * result.scale.repeat_interleave(
group_size, dim=1
)
torch.testing.assert_close(ref_deq, result_deq, atol=5e-2, rtol=5e-2)
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
@pytest.mark.parametrize("hidden_size", [128, 512])
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode()
def test_maybe_fused_act_quant_fallback(
default_vllm_config,
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
) -> None:
"""Test maybe_fused_act_quant falls back when no input_quant_key."""
device = "cuda:0"
torch.set_default_device(device)
act_fn = SiluAndMul()
linear = MockLinearNoQuant()
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device)
result = maybe_fused_act_quant(act_fn, x, linear)
assert isinstance(result, torch.Tensor)
assert not isinstance(result, QuantizedActivation)
assert result.dtype == dtype
assert result.shape == (num_tokens, hidden_size)
ref_out = act_fn(x)
torch.testing.assert_close(result, ref_out)