[Perf][Kernel][Quantization] Fuse ReLU2 with static FP8 activation quantization (#53793)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: ElizaWszola <ewszola@redhat.com> Signed-off-by: snordmann <snordmann@nvidia.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: ElizaWszola <ewszola@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mgoin
Claude Opus 4.8
ElizaWszola
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
435c96f9db
commit
03dc26e639
@@ -91,7 +91,7 @@ class MockLinearFp8Static(torch.nn.Module):
|
||||
|
||||
def __init__(self, input_scale: torch.Tensor):
|
||||
super().__init__()
|
||||
self.input_quant_key = kFp8StaticTensorSym
|
||||
self._input_quant_key = kFp8StaticTensorSym
|
||||
self.input_scale = input_scale
|
||||
|
||||
|
||||
@@ -100,15 +100,21 @@ class MockLinearFp8Dynamic128(torch.nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.input_quant_key = kFp8Dynamic128Sym
|
||||
self._input_quant_key = kFp8Dynamic128Sym
|
||||
|
||||
|
||||
class MockLinearNoQuant(torch.nn.Module):
|
||||
"""Mock linear layer with no input_quant_key (no fusion)."""
|
||||
"""Mock linear layer with no input quantization key."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class MockLinearRequiresUnquantized(MockLinearFp8Static):
|
||||
"""Mock consumer that needs the original activation for another branch."""
|
||||
|
||||
requires_unquantized_input = True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@@ -199,7 +205,7 @@ def test_maybe_fused_act_quant_fallback(
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Test maybe_fused_act_quant falls back when no input_quant_key."""
|
||||
"""Test maybe_fused_act_quant falls back without an input quantization key."""
|
||||
device = "cuda:0"
|
||||
torch.set_default_device(device)
|
||||
|
||||
@@ -216,3 +222,19 @@ def test_maybe_fused_act_quant_fallback(
|
||||
|
||||
ref_out = act_fn(x)
|
||||
torch.testing.assert_close(result, ref_out)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_maybe_fused_act_quant_preserves_required_unquantized_input(
|
||||
default_vllm_config,
|
||||
) -> None:
|
||||
device = "cuda:0"
|
||||
act_fn = SiluAndMul()
|
||||
scale = torch.tensor([0.5], device=device, dtype=torch.float32)
|
||||
linear = MockLinearRequiresUnquantized(scale)
|
||||
x = torch.randn(17, 256, dtype=torch.bfloat16, device=device)
|
||||
|
||||
result = maybe_fused_act_quant(act_fn, x, linear)
|
||||
|
||||
assert isinstance(result, torch.Tensor)
|
||||
torch.testing.assert_close(result, act_fn(x))
|
||||
|
||||
Reference in New Issue
Block a user