From 03dc26e639e1927cda51d7b62639a11a354e1c0d Mon Sep 17 00:00:00 2001 From: Samuel Nordmann Date: Mon, 14 Sep 2026 19:26:15 +0200 Subject: [PATCH] [Perf][Kernel][Quantization] Fuse ReLU2 with static FP8 activation quantization (#53793) Signed-off-by: mgoin Signed-off-by: ElizaWszola Signed-off-by: snordmann Co-authored-by: mgoin Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: ElizaWszola Co-authored-by: OpenAI Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../test_silu_mul_quant_manual_fusion.py | 11 +- .../fusion/test_quant_activation_contract.py | 5 +- tests/kernels/test_fused_quant_activation.py | 30 ++- tests/kernels/test_relu2_fp8_quant.py | 229 ++++++++++++++++++ tests/lora/test_layers.py | 10 + .../test_nemotron_h_quantization.py | 27 +++ vllm/lora/layers/base_linear.py | 4 + .../layers/fusion/fused_act_quant.py | 66 ++++- .../layers/fusion/quant_activation.py | 22 +- .../layers/fusion/relu2_fp8_quant.py | 76 ++++++ vllm/model_executor/layers/linear.py | 2 + vllm/model_executor/models/nemotron_h.py | 9 +- 12 files changed, 459 insertions(+), 32 deletions(-) create mode 100644 tests/kernels/test_relu2_fp8_quant.py create mode 100644 vllm/model_executor/layers/fusion/relu2_fp8_quant.py diff --git a/tests/compile/passes/test_silu_mul_quant_manual_fusion.py b/tests/compile/passes/test_silu_mul_quant_manual_fusion.py index 5ee749659b..ee94960844 100644 --- a/tests/compile/passes/test_silu_mul_quant_manual_fusion.py +++ b/tests/compile/passes/test_silu_mul_quant_manual_fusion.py @@ -34,6 +34,7 @@ from vllm.model_executor.layers.fusion.fused_act_quant import ( from vllm.model_executor.layers.fusion.quant_activation import ( QuantizedActivation, expose_input_quant_key, + get_input_quant_key, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Dynamic128Sym, @@ -45,7 +46,7 @@ from vllm.platforms import current_platform # Mock linear layer for testing fusion paths that don't have real kernel support class MockLinearForFusion(torch.nn.Module): - """Mock linear layer that exposes input_quant_key for fusion testing.""" + """Mock linear layer that exposes an input key for fusion testing.""" def __init__( self, @@ -55,7 +56,7 @@ class MockLinearForFusion(torch.nn.Module): input_global_scale_inv=None, ): super().__init__() - self.input_quant_key = quant_key + self._input_quant_key = quant_key if input_scale is not None: self.input_scale = input_scale if input_global_scale is not None: @@ -118,7 +119,7 @@ def test_manual_fusion_fp8_static_with_linear( # Enable fusion expose_input_quant_key(fp8_linear, fp8_linear.kernel) - if not hasattr(fp8_linear, "input_quant_key"): + if get_input_quant_key(fp8_linear) is None: pytest.skip( f"Kernel {force_kernel.__name__} doesn't support input_quant_key" ) @@ -315,7 +316,7 @@ def test_manual_fusion_nvfp4_dynamic( envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm" ) def test_manual_fusion_fallback_no_key(dtype: torch.dtype): - """Test that maybe_fused_act_quant falls back when no input_quant_key.""" + """Test that maybe_fused_act_quant falls back without an input key.""" torch.set_default_device("cuda") torch.set_default_dtype(dtype) @@ -327,7 +328,7 @@ def test_manual_fusion_fallback_no_key(dtype: torch.dtype): with set_current_vllm_config(config): silu_and_mul = SiluAndMul() - # Linear without input_quant_key attribute + # Linear without an input quantization key mock_linear = torch.nn.Linear(128, 128) result = maybe_fused_act_quant(silu_and_mul, x, mock_linear) diff --git a/tests/fusion/test_quant_activation_contract.py b/tests/fusion/test_quant_activation_contract.py index f698a272f5..ab12393692 100644 --- a/tests/fusion/test_quant_activation_contract.py +++ b/tests/fusion/test_quant_activation_contract.py @@ -42,6 +42,7 @@ from vllm.model_executor.layers.fusion.quant_activation import ( QuantizedActivation, as_quantized_activation, expose_input_quant_key, + get_input_quant_key, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticTensorSym, @@ -118,13 +119,13 @@ def test_bridge_marks_supporting_and_skips_others(): supported = _probe(FlashInferCutlassNvFp4LinearKernel) layer = torch.nn.Module() expose_input_quant_key(layer, supported) - assert layer.input_quant_key == kNvfp4Dynamic + assert get_input_quant_key(layer) == kNvfp4Dynamic unsupported = _probe(FlashInferTrtllmNvFp4LinearKernel) assert unsupported.input_quant_key() is None layer = torch.nn.Module() expose_input_quant_key(layer, unsupported) - assert not hasattr(layer, "input_quant_key") + assert get_input_quant_key(layer) is None def test_as_quantized_activation_validates_key(): diff --git a/tests/kernels/test_fused_quant_activation.py b/tests/kernels/test_fused_quant_activation.py index 267484e433..321cb6978e 100644 --- a/tests/kernels/test_fused_quant_activation.py +++ b/tests/kernels/test_fused_quant_activation.py @@ -91,7 +91,7 @@ class MockLinearFp8Static(torch.nn.Module): def __init__(self, input_scale: torch.Tensor): super().__init__() - self.input_quant_key = kFp8StaticTensorSym + self._input_quant_key = kFp8StaticTensorSym self.input_scale = input_scale @@ -100,15 +100,21 @@ class MockLinearFp8Dynamic128(torch.nn.Module): def __init__(self): super().__init__() - self.input_quant_key = kFp8Dynamic128Sym + self._input_quant_key = kFp8Dynamic128Sym class MockLinearNoQuant(torch.nn.Module): - """Mock linear layer with no input_quant_key (no fusion).""" + """Mock linear layer with no input quantization key.""" pass +class MockLinearRequiresUnquantized(MockLinearFp8Static): + """Mock consumer that needs the original activation for another branch.""" + + requires_unquantized_input = True + + @pytest.mark.parametrize("num_tokens", [1, 16, 128]) @pytest.mark.parametrize("hidden_size", [128, 512, 1024]) @pytest.mark.parametrize("dtype", DTYPES) @@ -199,7 +205,7 @@ def test_maybe_fused_act_quant_fallback( hidden_size: int, dtype: torch.dtype, ) -> None: - """Test maybe_fused_act_quant falls back when no input_quant_key.""" + """Test maybe_fused_act_quant falls back without an input quantization key.""" device = "cuda:0" torch.set_default_device(device) @@ -216,3 +222,19 @@ def test_maybe_fused_act_quant_fallback( ref_out = act_fn(x) torch.testing.assert_close(result, ref_out) + + +@torch.inference_mode() +def test_maybe_fused_act_quant_preserves_required_unquantized_input( + default_vllm_config, +) -> None: + device = "cuda:0" + act_fn = SiluAndMul() + scale = torch.tensor([0.5], device=device, dtype=torch.float32) + linear = MockLinearRequiresUnquantized(scale) + x = torch.randn(17, 256, dtype=torch.bfloat16, device=device) + + result = maybe_fused_act_quant(act_fn, x, linear) + + assert isinstance(result, torch.Tensor) + torch.testing.assert_close(result, act_fn(x)) diff --git a/tests/kernels/test_relu2_fp8_quant.py b/tests/kernels/test_relu2_fp8_quant.py new file mode 100644 index 0000000000..cb605914c7 --- /dev/null +++ b/tests/kernels/test_relu2_fp8_quant.py @@ -0,0 +1,229 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config +from vllm.config.vllm import OptimizationLevel +from vllm.model_executor.layers.activation import ReLUSquaredActivation +from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant +from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation +from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8 +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + GroupShape, + kFp8StaticTensorSym, +) +from vllm.platforms import current_platform + +requires_sm90 = pytest.mark.skipif( + not (current_platform.is_cuda() and current_platform.has_device_capability(90)), + reason="This test requires SM90 or newer", +) + + +class _StaticFp8Linear(torch.nn.Module): + def __init__(self, input_scale: torch.Tensor) -> None: + super().__init__() + self._input_quant_key = kFp8StaticTensorSym + self.input_scale = input_scale + + +def _assert_fp8_bitwise_equal(actual: torch.Tensor, expected: torch.Tensor) -> None: + assert actual.dtype == expected.dtype == current_platform.fp8_dtype() + torch.testing.assert_close( + actual.view(torch.uint8), expected.view(torch.uint8), rtol=0, atol=0 + ) + + +def _assert_quantized_activation( + result: QuantizedActivation, x: torch.Tensor, scale: torch.Tensor +) -> None: + assert result.scale is scale + assert result.orig_dtype == x.dtype + assert result.orig_shape == x.shape + assert result.quant_key == kFp8StaticTensorSym + + +@pytest.fixture(scope="module") +def o2_relu2_fp8_ops(): + config = VllmConfig( + optimization_level=OptimizationLevel.O2, + compilation_config=CompilationConfig(custom_ops=["none"]), + ) + with set_current_vllm_config(config): + yield ( + ReLUSquaredActivation(), + QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR), + ) + + +@requires_sm90 +@pytest.mark.parametrize("shape", [(1, 1), (1, 16), (17, 5120), (128, 10240)]) +@torch.inference_mode() +def test_relu2_static_fp8_quant(o2_relu2_fp8_ops, shape) -> None: + torch.manual_seed(0) + x = torch.randn(shape, device="cuda", dtype=torch.bfloat16) + scale = torch.tensor(0.1495535671710968, device="cuda", dtype=torch.float32) + + relu2, quant_fp8 = o2_relu2_fp8_ops + expected, _ = quant_fp8(relu2(x), scale) + result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale)) + + assert isinstance(result, QuantizedActivation) + _assert_quantized_activation(result, x, scale) + _assert_fp8_bitwise_equal(result.data, expected) + + +@requires_sm90 +@pytest.mark.parametrize( + "enabled_ops", + [(), ("relu2",), ("quant_fp8",), ("relu2", "quant_fp8")], +) +@torch.inference_mode() +def test_relu2_static_fp8_quant_ignores_custom_op_dispatch(enabled_ops) -> None: + custom_ops = ["none", *(f"+{op}" for op in enabled_ops)] + config = VllmConfig( + optimization_level=OptimizationLevel.O2, + compilation_config=CompilationConfig(custom_ops=custom_ops), + ) + with set_current_vllm_config(config): + torch.manual_seed(0) + relu2 = ReLUSquaredActivation() + quant_fp8 = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR) + x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16) + scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32) + + expected, _ = quant_fp8(relu2(x), scale) + result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale)) + + assert isinstance(result, QuantizedActivation) + _assert_quantized_activation(result, x, scale) + _assert_fp8_bitwise_equal(result.data, expected) + + +@requires_sm90 +@torch.inference_mode() +def test_relu2_static_fp8_quant_without_compilation() -> None: + config = VllmConfig(optimization_level=OptimizationLevel.O0) + with set_current_vllm_config(config): + torch.manual_seed(0) + relu2 = ReLUSquaredActivation() + quant_fp8 = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR) + x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16) + scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32) + + expected, _ = quant_fp8(relu2(x), scale) + result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale)) + + assert isinstance(result, QuantizedActivation) + _assert_quantized_activation(result, x, scale) + _assert_fp8_bitwise_equal(result.data, expected) + + +@requires_sm90 +@pytest.mark.parametrize( + "scale_value", + [ + 0.00435965, + 0.007149832788854837, + 0.03125, + 0.0714285746216774, + 0.1495535671710968, + 0.1517857164144516, + 6.03571415, + ], +) +@torch.inference_mode() +def test_relu2_static_fp8_quant_all_bf16_values(o2_relu2_fp8_ops, scale_value) -> None: + bits = torch.arange(1 << 16, dtype=torch.int32).to(torch.uint16) + x = bits.view(torch.bfloat16).reshape(1, -1).to("cuda") + scale = torch.tensor(scale_value, device="cuda", dtype=torch.float32) + + relu2, quant_fp8 = o2_relu2_fp8_ops + expected, _ = quant_fp8(relu2(x), scale) + result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale)) + + assert isinstance(result, QuantizedActivation) + _assert_quantized_activation(result, x, scale) + _assert_fp8_bitwise_equal(result.data, expected) + + +@requires_sm90 +@torch.inference_mode() +def test_relu2_static_fp8_quant_empty(o2_relu2_fp8_ops) -> None: + x = torch.empty(0, device="cuda", dtype=torch.bfloat16) + scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32) + + relu2, _ = o2_relu2_fp8_ops + result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale)) + + assert isinstance(result, QuantizedActivation) + _assert_quantized_activation(result, x, scale) + assert result.data.shape == x.shape + assert result.data.dtype == current_platform.fp8_dtype() + + +@requires_sm90 +@pytest.mark.parametrize("unsupported", ["dtype", "layout", "scale"]) +@torch.inference_mode() +def test_relu2_static_fp8_quant_falls_back(o2_relu2_fp8_ops, unsupported: str) -> None: + x = torch.randn((17, 32), device="cuda", dtype=torch.bfloat16) + scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32) + if unsupported == "dtype": + x = x.to(torch.float16) + elif unsupported == "layout": + x = x.T + else: + scale = scale.to(torch.float16) + + relu2, _ = o2_relu2_fp8_ops + expected = relu2(x) + result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale)) + + assert isinstance(result, torch.Tensor) + assert not isinstance(result, QuantizedActivation) + torch.testing.assert_close(result, expected, rtol=0, atol=0) + + +@requires_sm90 +@torch.inference_mode() +def test_relu2_static_fp8_quant_torch_compile_dynamic( + o2_relu2_fp8_ops, +) -> None: + torch.manual_seed(1) + scale = torch.tensor(0.00435965, device="cuda", dtype=torch.float32) + relu2, quant_fp8 = o2_relu2_fp8_ops + linear = _StaticFp8Linear(scale) + + def fused(x: torch.Tensor) -> torch.Tensor: + result = maybe_fused_act_quant(relu2, x, linear) + assert isinstance(result, QuantizedActivation) + return result.data + + compile_count = 0 + + def counting_inductor_backend(gm, example_inputs): + nonlocal compile_count + compile_count += 1 + return torch._inductor.compile(gm, example_inputs) + + torch._dynamo.reset() + try: + x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16) + torch._dynamo.mark_dynamic(x, 0) + compiled_fused = torch.compile( + fused, backend=counting_inductor_backend, fullgraph=True + ) + + for candidate in ( + x, + torch.randn((31, 5120), device="cuda", dtype=torch.bfloat16), + torch.randn((63, 5120), device="cuda", dtype=torch.bfloat16), + ): + expected, _ = quant_fp8(relu2(candidate), scale) + _assert_fp8_bitwise_equal(compiled_fused(candidate), expected) + + assert compile_count == 1 + finally: + torch._dynamo.reset() diff --git a/tests/lora/test_layers.py b/tests/lora/test_layers.py index bce91510d4..2dab86bc0d 100644 --- a/tests/lora/test_layers.py +++ b/tests/lora/test_layers.py @@ -31,6 +31,9 @@ from vllm.lora.layers import ( ) from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights from vllm.lora.punica_wrapper import get_punica_wrapper +from vllm.model_executor.layers.fusion.quant_activation import ( + get_input_quant_key, +) from vllm.model_executor.layers.linear import ( ColumnParallelLinear, MergedColumnParallelLinear, @@ -56,6 +59,13 @@ TOLERANCES = { torch.bfloat16: (3e-2, 2e-2), } + +def test_lora_linear_requires_unquantized_input() -> None: + layer = RowParallelLinearWithLoRA.__new__(RowParallelLinearWithLoRA) + layer._input_quant_key = object() + assert get_input_quant_key(layer) is None + + pytestmark = [ pytest.mark.skipif( not ( diff --git a/tests/model_executor/test_nemotron_h_quantization.py b/tests/model_executor/test_nemotron_h_quantization.py index 81a3fb5b13..e4e1c87db4 100644 --- a/tests/model_executor/test_nemotron_h_quantization.py +++ b/tests/model_executor/test_nemotron_h_quantization.py @@ -3,6 +3,8 @@ from unittest.mock import Mock, patch +import torch + def test_nemotron_h_lm_head_receives_quant_config(): from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM @@ -32,3 +34,28 @@ def test_nemotron_h_lm_head_receives_quant_config(): MockLMHead.assert_called_once() call_kwargs = MockLMHead.call_args.kwargs assert call_kwargs["quant_config"] is mock_quant_config + + +def test_relu2_fp8_fusion_uses_registry(): + from vllm.model_executor.models.nemotron_h import NemotronHMLP + + projected = torch.empty((1, 1), dtype=torch.bfloat16) + fused = Mock() + act_fn = Mock() + down_proj = Mock(side_effect=lambda x: (x, None)) + + mlp = NemotronHMLP.__new__(NemotronHMLP) + torch.nn.Module.__init__(mlp) + mlp.up_proj = Mock(return_value=(projected, None)) + mlp.down_proj = down_proj + mlp.act_fn = act_fn + + with patch( + "vllm.model_executor.models.nemotron_h.maybe_fused_act_quant", + return_value=fused, + ) as maybe_fused: + result = mlp(Mock()) + + maybe_fused.assert_called_once_with(act_fn, projected, down_proj) + act_fn.assert_not_called() + assert result is fused diff --git a/vllm/lora/layers/base_linear.py b/vllm/lora/layers/base_linear.py index 5c8e829b29..df7cfe7122 100644 --- a/vllm/lora/layers/base_linear.py +++ b/vllm/lora/layers/base_linear.py @@ -68,6 +68,10 @@ if envs.VLLM_LORA_ENABLE_DUAL_STREAM: class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): + # The adapter branch consumes the original activation, independently of + # whether the wrapped base layer can consume a pre-quantized activation. + requires_unquantized_input = True + def __init__(self, base_layer: LinearBase): super().__init__() diff --git a/vllm/model_executor/layers/fusion/fused_act_quant.py b/vllm/model_executor/layers/fusion/fused_act_quant.py index 9fe253f2d9..e18a35a70d 100644 --- a/vllm/model_executor/layers/fusion/fused_act_quant.py +++ b/vllm/model_executor/layers/fusion/fused_act_quant.py @@ -5,21 +5,27 @@ Producer side of the QuantizedActivation contract for activation layers. Given an activation module and the downstream linear it feeds, fuse the activation with that linear's input quantization into a single kernel when the -linear advertises a consumable input_quant_key (see quant_activation.py). +linear advertises a consumable input quantization key (see quant_activation.py). Falls back to the plain activation when nothing matches, so a model forward can always call maybe_fused_act_quant unconditionally. This is the manual-fusion counterpart to ActivationQuantFusionPass: when fusion -fires here the silu_and_mul pattern is already consumed, so the compiler pass -finds nothing to rewrite and the two never double-fuse. +fires here, the activation and quantization are already consumed, so a compiler +pass cannot fuse the same boundary again. """ from collections.abc import Callable import torch -from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation +from vllm.model_executor.layers.activation import ReLUSquaredActivation, SiluAndMul +from vllm.model_executor.layers.fusion.quant_activation import ( + QuantizedActivation, + get_input_quant_key, +) +from vllm.model_executor.layers.fusion.relu2_fp8_quant import ( + relu_squared_static_fp8_quant, +) from vllm.model_executor.layers.linear import LinearBase from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, @@ -139,12 +145,46 @@ def _silu_and_mul_nvfp4_dynamic( ) -# (activation module type, consumer input_quant_key) -> fused producer. -# Mirrors ActivationQuantFusionPass.FUSED_OPS; add a row to migrate a scheme. +# (activation module type, consumer input quantization key) -> fused producer. +# Add a row for each supported manual producer. _FUSED_ACT_QUANT: dict[tuple[type, QuantKey], Callable] = { (SiluAndMul, kFp8StaticTensorSym): _silu_and_mul_fp8_static, } + +def _relu_squared_static_fp8_quant_supported( + act_fn: torch.nn.Module, + x: torch.Tensor, + linear: LinearBase, +) -> bool: + """Return whether the ReLU2 static-FP8 producer can consume this input.""" + scale = getattr(linear, "input_scale", None) + return ( + isinstance(act_fn, ReLUSquaredActivation) + and x.is_cuda + and x.dtype == torch.bfloat16 + and x.is_contiguous() + and isinstance(scale, torch.Tensor) + and scale.dtype == torch.float32 + and scale.device == x.device + and scale.numel() == 1 + ) + + +# Optional per-entry predicates preserve the unconditional fallback contract +# for producers whose supported inputs are narrower than their registry key. +_FUSED_ACT_QUANT_SUPPORT: dict[ + tuple[type, QuantKey], Callable[[torch.nn.Module, torch.Tensor, LinearBase], bool] +] = {} + +# This path is validated on SM90 and newer CUDA devices. +if current_platform.is_cuda() and current_platform.has_device_capability(90): + _RELU2_STATIC_FP8_KEY = (ReLUSquaredActivation, kFp8StaticTensorSym) + _FUSED_ACT_QUANT[_RELU2_STATIC_FP8_KEY] = relu_squared_static_fp8_quant + _FUSED_ACT_QUANT_SUPPORT[_RELU2_STATIC_FP8_KEY] = ( + _relu_squared_static_fp8_quant_supported + ) + # Add CUDA-specific entries for dynamic block quantization if current_platform.is_cuda_alike(): _FUSED_ACT_QUANT[(SiluAndMul, kFp8Dynamic128Sym)] = _silu_and_mul_fp8_dynamic_128 @@ -161,12 +201,14 @@ def maybe_fused_act_quant( ) -> "torch.Tensor | QuantizedActivation": """Apply act_fn, fusing the downstream linear's input quant when possible. - Returns a QuantizedActivation when a fused kernel matches - (act_fn, linear.input_quant_key), else the plain activated tensor. + Returns a QuantizedActivation when a fused kernel matches the activation and + the consumer's effective input quantization key, else the plain activation. """ - key = getattr(linear, "input_quant_key", None) + key = get_input_quant_key(linear) if key is not None: - producer = _FUSED_ACT_QUANT.get((type(act_fn), key)) - if producer is not None: + registry_key = (type(act_fn), key) + producer = _FUSED_ACT_QUANT.get(registry_key) + support = _FUSED_ACT_QUANT_SUPPORT.get(registry_key) + if producer is not None and (support is None or support(act_fn, x, linear)): return producer(x, linear) return act_fn(x) diff --git a/vllm/model_executor/layers/fusion/quant_activation.py b/vllm/model_executor/layers/fusion/quant_activation.py index bd6cab668e..d69415448f 100644 --- a/vllm/model_executor/layers/fusion/quant_activation.py +++ b/vllm/model_executor/layers/fusion/quant_activation.py @@ -5,7 +5,9 @@ A QuantizedActivation is a pre-quantized activation produced by a fused kernel and consumed directly by a linear layer, letting the layer skip its own input quantization. A linear advertises the key its kernel can consume via expose_input_quant_key; the kernel validates and reads the activation via -as_quantized_activation. +as_quantized_activation. Producers query the effective capability through +get_input_quant_key, which hides the key when another consumer branch needs the +original activation. """ from dataclasses import dataclass, replace @@ -46,12 +48,11 @@ class QuantizedActivation: def expose_input_quant_key(layer: torch.nn.Module, kernel) -> None: - """Advertise the kernel's pre-quantized input key on the layer, if any. + """Store the kernel's pre-quantized input key on the layer, if any. - This is the bridge from a kernel's input_quant_key() to the - layer.input_quant_key attribute that fusion call sites read. The attribute - is left unset when the kernel quantizes its own input, so non-supporting - backends never receive a QuantizedActivation. + This is the bridge from a kernel's input_quant_key() to the layer capability + that fusion call sites read through get_input_quant_key. The raw key is left + unset when the kernel quantizes its own input. TODO(mgoin): Producers also need the consumer's quantization scales (e.g. static input scale, global scale). Expose those here as well so producers @@ -59,7 +60,14 @@ def expose_input_quant_key(layer: torch.nn.Module, kernel) -> None: """ key = kernel.input_quant_key() if key is not None: - layer.input_quant_key = key + layer._input_quant_key = key + + +def get_input_quant_key(layer: torch.nn.Module) -> QuantKey | None: + """Return the pre-quantized input key the complete layer can consume.""" + if getattr(layer, "requires_unquantized_input", False): + return None + return getattr(layer, "_input_quant_key", None) def as_quantized_activation( diff --git a/vllm/model_executor/layers/fusion/relu2_fp8_quant.py b/vllm/model_executor/layers/fusion/relu2_fp8_quant.py new file mode 100644 index 0000000000..4dc0fed4cc --- /dev/null +++ b/vllm/model_executor/layers/fusion/relu2_fp8_quant.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation +from vllm.model_executor.layers.linear import LinearBase +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + get_fp8_min_max, + kFp8StaticTensorSym, +) +from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton + +_FP8_MIN, _FP8_MAX = get_fp8_min_max() + + +@triton.jit +def _relu_squared_static_fp8_quant_kernel( + x_ptr, + scale_ptr, + output_ptr, + num_elements, + FP8_MIN: tl.constexpr, + FP8_MAX: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = offsets < num_elements + x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32) + input_is_nan = x != x + relu = tl.maximum(x, 0.0) + # Preserve the standalone ReLU2 kernel's BF16 output boundary. + activated = (relu * relu).to(tl.bfloat16).to(tl.float32) + # Match O2's Inductor-compiled QuantFP8 reciprocal-then-multiply. + quantized = activated * (1.0 / tl.load(scale_ptr)) + quantized = tl.maximum(tl.minimum(quantized, FP8_MAX), FP8_MIN) + # Match the O2 native chain by preserving NaN through the FP8 conversion. + quantized = tl.where(input_is_nan, x, quantized) + tl.store(output_ptr + offsets, quantized, mask=mask) + + +def relu_squared_static_fp8_quant( + x: torch.Tensor, linear: LinearBase +) -> QuantizedActivation: + """BF16 ReLU2 followed by static per-tensor FP8 quantization.""" + assert x.dtype == torch.bfloat16 + assert x.is_contiguous() + scale = linear.input_scale + assert scale.dtype == torch.float32 + assert x.device == scale.device + assert scale.numel() == 1 + + output = torch.empty_like(x, dtype=current_platform.fp8_dtype()) + if x.numel() != 0: + block_size = min(triton.next_power_of_2(x.shape[-1]), 2048) + num_warps = min(max(block_size // 256, 1), 4) + grid = lambda meta: (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),) + _relu_squared_static_fp8_quant_kernel[grid]( + x, + scale, + output, + x.numel(), + FP8_MIN=_FP8_MIN, + FP8_MAX=_FP8_MAX, + BLOCK_SIZE=block_size, + num_warps=num_warps, + ) + + return QuantizedActivation( + data=output, + scale=scale, + orig_dtype=x.dtype, + orig_shape=x.shape, + quant_key=kFp8StaticTensorSym, + ) diff --git a/vllm/model_executor/layers/linear.py b/vllm/model_executor/layers/linear.py index 212dd95375..95dc852fb3 100644 --- a/vllm/model_executor/layers/linear.py +++ b/vllm/model_executor/layers/linear.py @@ -252,6 +252,8 @@ class LinearBase(PluggableLayer): disable_tp: If true, tensor parallelism will be disabled for this layer. """ + requires_unquantized_input = False + def __init__( self, input_size: int, diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index e65e51f3bb..c1bc4c6823 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -25,7 +25,11 @@ import torch from torch import nn from vllm.compilation.decorators import support_torch_compile -from vllm.config import CacheConfig, ModelConfig, VllmConfig +from vllm.config import ( + CacheConfig, + ModelConfig, + VllmConfig, +) from vllm.config.parallel import ParallelConfig from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size from vllm.distributed.communication_op import tensor_model_parallel_all_gather @@ -38,6 +42,7 @@ from vllm.model_executor.layers.fused_moe import ( activation_without_mul, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -118,7 +123,7 @@ class NemotronHMLP(nn.Module): def forward(self, x: torch.Tensor): x, _ = self.up_proj(x) - x = self.act_fn(x) + x = maybe_fused_act_quant(self.act_fn, x, self.down_proj) x, _ = self.down_proj(x) return x