[Perf][Kernel][Quantization] Fuse ReLU2 with static FP8 activation quantization (#53793)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: ElizaWszola <ewszola@redhat.com> Signed-off-by: snordmann <snordmann@nvidia.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: ElizaWszola <ewszola@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mgoin
Claude Opus 4.8
ElizaWszola
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
435c96f9db
commit
03dc26e639
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.fusion.fused_act_quant import (
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
expose_input_quant_key,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
@@ -45,7 +46,7 @@ from vllm.platforms import current_platform
|
||||
|
||||
# Mock linear layer for testing fusion paths that don't have real kernel support
|
||||
class MockLinearForFusion(torch.nn.Module):
|
||||
"""Mock linear layer that exposes input_quant_key for fusion testing."""
|
||||
"""Mock linear layer that exposes an input key for fusion testing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -55,7 +56,7 @@ class MockLinearForFusion(torch.nn.Module):
|
||||
input_global_scale_inv=None,
|
||||
):
|
||||
super().__init__()
|
||||
self.input_quant_key = quant_key
|
||||
self._input_quant_key = quant_key
|
||||
if input_scale is not None:
|
||||
self.input_scale = input_scale
|
||||
if input_global_scale is not None:
|
||||
@@ -118,7 +119,7 @@ def test_manual_fusion_fp8_static_with_linear(
|
||||
# Enable fusion
|
||||
expose_input_quant_key(fp8_linear, fp8_linear.kernel)
|
||||
|
||||
if not hasattr(fp8_linear, "input_quant_key"):
|
||||
if get_input_quant_key(fp8_linear) is None:
|
||||
pytest.skip(
|
||||
f"Kernel {force_kernel.__name__} doesn't support input_quant_key"
|
||||
)
|
||||
@@ -315,7 +316,7 @@ def test_manual_fusion_nvfp4_dynamic(
|
||||
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
|
||||
)
|
||||
def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
|
||||
"""Test that maybe_fused_act_quant falls back when no input_quant_key."""
|
||||
"""Test that maybe_fused_act_quant falls back without an input key."""
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
@@ -327,7 +328,7 @@ def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
silu_and_mul = SiluAndMul()
|
||||
# Linear without input_quant_key attribute
|
||||
# Linear without an input quantization key
|
||||
mock_linear = torch.nn.Linear(128, 128)
|
||||
|
||||
result = maybe_fused_act_quant(silu_and_mul, x, mock_linear)
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
as_quantized_activation,
|
||||
expose_input_quant_key,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
@@ -118,13 +119,13 @@ def test_bridge_marks_supporting_and_skips_others():
|
||||
supported = _probe(FlashInferCutlassNvFp4LinearKernel)
|
||||
layer = torch.nn.Module()
|
||||
expose_input_quant_key(layer, supported)
|
||||
assert layer.input_quant_key == kNvfp4Dynamic
|
||||
assert get_input_quant_key(layer) == kNvfp4Dynamic
|
||||
|
||||
unsupported = _probe(FlashInferTrtllmNvFp4LinearKernel)
|
||||
assert unsupported.input_quant_key() is None
|
||||
layer = torch.nn.Module()
|
||||
expose_input_quant_key(layer, unsupported)
|
||||
assert not hasattr(layer, "input_quant_key")
|
||||
assert get_input_quant_key(layer) is None
|
||||
|
||||
|
||||
def test_as_quantized_activation_validates_key():
|
||||
|
||||
@@ -91,7 +91,7 @@ class MockLinearFp8Static(torch.nn.Module):
|
||||
|
||||
def __init__(self, input_scale: torch.Tensor):
|
||||
super().__init__()
|
||||
self.input_quant_key = kFp8StaticTensorSym
|
||||
self._input_quant_key = kFp8StaticTensorSym
|
||||
self.input_scale = input_scale
|
||||
|
||||
|
||||
@@ -100,15 +100,21 @@ class MockLinearFp8Dynamic128(torch.nn.Module):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.input_quant_key = kFp8Dynamic128Sym
|
||||
self._input_quant_key = kFp8Dynamic128Sym
|
||||
|
||||
|
||||
class MockLinearNoQuant(torch.nn.Module):
|
||||
"""Mock linear layer with no input_quant_key (no fusion)."""
|
||||
"""Mock linear layer with no input quantization key."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class MockLinearRequiresUnquantized(MockLinearFp8Static):
|
||||
"""Mock consumer that needs the original activation for another branch."""
|
||||
|
||||
requires_unquantized_input = True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@@ -199,7 +205,7 @@ def test_maybe_fused_act_quant_fallback(
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Test maybe_fused_act_quant falls back when no input_quant_key."""
|
||||
"""Test maybe_fused_act_quant falls back without an input quantization key."""
|
||||
device = "cuda:0"
|
||||
torch.set_default_device(device)
|
||||
|
||||
@@ -216,3 +222,19 @@ def test_maybe_fused_act_quant_fallback(
|
||||
|
||||
ref_out = act_fn(x)
|
||||
torch.testing.assert_close(result, ref_out)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_maybe_fused_act_quant_preserves_required_unquantized_input(
|
||||
default_vllm_config,
|
||||
) -> None:
|
||||
device = "cuda:0"
|
||||
act_fn = SiluAndMul()
|
||||
scale = torch.tensor([0.5], device=device, dtype=torch.float32)
|
||||
linear = MockLinearRequiresUnquantized(scale)
|
||||
x = torch.randn(17, 256, dtype=torch.bfloat16, device=device)
|
||||
|
||||
result = maybe_fused_act_quant(act_fn, x, linear)
|
||||
|
||||
assert isinstance(result, torch.Tensor)
|
||||
torch.testing.assert_close(result, act_fn(x))
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.vllm import OptimizationLevel
|
||||
from vllm.model_executor.layers.activation import ReLUSquaredActivation
|
||||
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
requires_sm90 = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() and current_platform.has_device_capability(90)),
|
||||
reason="This test requires SM90 or newer",
|
||||
)
|
||||
|
||||
|
||||
class _StaticFp8Linear(torch.nn.Module):
|
||||
def __init__(self, input_scale: torch.Tensor) -> None:
|
||||
super().__init__()
|
||||
self._input_quant_key = kFp8StaticTensorSym
|
||||
self.input_scale = input_scale
|
||||
|
||||
|
||||
def _assert_fp8_bitwise_equal(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
assert actual.dtype == expected.dtype == current_platform.fp8_dtype()
|
||||
torch.testing.assert_close(
|
||||
actual.view(torch.uint8), expected.view(torch.uint8), rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
def _assert_quantized_activation(
|
||||
result: QuantizedActivation, x: torch.Tensor, scale: torch.Tensor
|
||||
) -> None:
|
||||
assert result.scale is scale
|
||||
assert result.orig_dtype == x.dtype
|
||||
assert result.orig_shape == x.shape
|
||||
assert result.quant_key == kFp8StaticTensorSym
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def o2_relu2_fp8_ops():
|
||||
config = VllmConfig(
|
||||
optimization_level=OptimizationLevel.O2,
|
||||
compilation_config=CompilationConfig(custom_ops=["none"]),
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
yield (
|
||||
ReLUSquaredActivation(),
|
||||
QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR),
|
||||
)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@pytest.mark.parametrize("shape", [(1, 1), (1, 16), (17, 5120), (128, 10240)])
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant(o2_relu2_fp8_ops, shape) -> None:
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(shape, device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor(0.1495535671710968, device="cuda", dtype=torch.float32)
|
||||
|
||||
relu2, quant_fp8 = o2_relu2_fp8_ops
|
||||
expected, _ = quant_fp8(relu2(x), scale)
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
_assert_quantized_activation(result, x, scale)
|
||||
_assert_fp8_bitwise_equal(result.data, expected)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@pytest.mark.parametrize(
|
||||
"enabled_ops",
|
||||
[(), ("relu2",), ("quant_fp8",), ("relu2", "quant_fp8")],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_ignores_custom_op_dispatch(enabled_ops) -> None:
|
||||
custom_ops = ["none", *(f"+{op}" for op in enabled_ops)]
|
||||
config = VllmConfig(
|
||||
optimization_level=OptimizationLevel.O2,
|
||||
compilation_config=CompilationConfig(custom_ops=custom_ops),
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
torch.manual_seed(0)
|
||||
relu2 = ReLUSquaredActivation()
|
||||
quant_fp8 = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
|
||||
|
||||
expected, _ = quant_fp8(relu2(x), scale)
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
_assert_quantized_activation(result, x, scale)
|
||||
_assert_fp8_bitwise_equal(result.data, expected)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_without_compilation() -> None:
|
||||
config = VllmConfig(optimization_level=OptimizationLevel.O0)
|
||||
with set_current_vllm_config(config):
|
||||
torch.manual_seed(0)
|
||||
relu2 = ReLUSquaredActivation()
|
||||
quant_fp8 = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
|
||||
|
||||
expected, _ = quant_fp8(relu2(x), scale)
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
_assert_quantized_activation(result, x, scale)
|
||||
_assert_fp8_bitwise_equal(result.data, expected)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@pytest.mark.parametrize(
|
||||
"scale_value",
|
||||
[
|
||||
0.00435965,
|
||||
0.007149832788854837,
|
||||
0.03125,
|
||||
0.0714285746216774,
|
||||
0.1495535671710968,
|
||||
0.1517857164144516,
|
||||
6.03571415,
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_all_bf16_values(o2_relu2_fp8_ops, scale_value) -> None:
|
||||
bits = torch.arange(1 << 16, dtype=torch.int32).to(torch.uint16)
|
||||
x = bits.view(torch.bfloat16).reshape(1, -1).to("cuda")
|
||||
scale = torch.tensor(scale_value, device="cuda", dtype=torch.float32)
|
||||
|
||||
relu2, quant_fp8 = o2_relu2_fp8_ops
|
||||
expected, _ = quant_fp8(relu2(x), scale)
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
_assert_quantized_activation(result, x, scale)
|
||||
_assert_fp8_bitwise_equal(result.data, expected)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_empty(o2_relu2_fp8_ops) -> None:
|
||||
x = torch.empty(0, device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
|
||||
|
||||
relu2, _ = o2_relu2_fp8_ops
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
_assert_quantized_activation(result, x, scale)
|
||||
assert result.data.shape == x.shape
|
||||
assert result.data.dtype == current_platform.fp8_dtype()
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@pytest.mark.parametrize("unsupported", ["dtype", "layout", "scale"])
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_falls_back(o2_relu2_fp8_ops, unsupported: str) -> None:
|
||||
x = torch.randn((17, 32), device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
|
||||
if unsupported == "dtype":
|
||||
x = x.to(torch.float16)
|
||||
elif unsupported == "layout":
|
||||
x = x.T
|
||||
else:
|
||||
scale = scale.to(torch.float16)
|
||||
|
||||
relu2, _ = o2_relu2_fp8_ops
|
||||
expected = relu2(x)
|
||||
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
|
||||
|
||||
assert isinstance(result, torch.Tensor)
|
||||
assert not isinstance(result, QuantizedActivation)
|
||||
torch.testing.assert_close(result, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
@torch.inference_mode()
|
||||
def test_relu2_static_fp8_quant_torch_compile_dynamic(
|
||||
o2_relu2_fp8_ops,
|
||||
) -> None:
|
||||
torch.manual_seed(1)
|
||||
scale = torch.tensor(0.00435965, device="cuda", dtype=torch.float32)
|
||||
relu2, quant_fp8 = o2_relu2_fp8_ops
|
||||
linear = _StaticFp8Linear(scale)
|
||||
|
||||
def fused(x: torch.Tensor) -> torch.Tensor:
|
||||
result = maybe_fused_act_quant(relu2, x, linear)
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
return result.data
|
||||
|
||||
compile_count = 0
|
||||
|
||||
def counting_inductor_backend(gm, example_inputs):
|
||||
nonlocal compile_count
|
||||
compile_count += 1
|
||||
return torch._inductor.compile(gm, example_inputs)
|
||||
|
||||
torch._dynamo.reset()
|
||||
try:
|
||||
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
compiled_fused = torch.compile(
|
||||
fused, backend=counting_inductor_backend, fullgraph=True
|
||||
)
|
||||
|
||||
for candidate in (
|
||||
x,
|
||||
torch.randn((31, 5120), device="cuda", dtype=torch.bfloat16),
|
||||
torch.randn((63, 5120), device="cuda", dtype=torch.bfloat16),
|
||||
):
|
||||
expected, _ = quant_fp8(relu2(candidate), scale)
|
||||
_assert_fp8_bitwise_equal(compiled_fused(candidate), expected)
|
||||
|
||||
assert compile_count == 1
|
||||
finally:
|
||||
torch._dynamo.reset()
|
||||
@@ -31,6 +31,9 @@ from vllm.lora.layers import (
|
||||
)
|
||||
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
|
||||
from vllm.lora.punica_wrapper import get_punica_wrapper
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
@@ -56,6 +59,13 @@ TOLERANCES = {
|
||||
torch.bfloat16: (3e-2, 2e-2),
|
||||
}
|
||||
|
||||
|
||||
def test_lora_linear_requires_unquantized_input() -> None:
|
||||
layer = RowParallelLinearWithLoRA.__new__(RowParallelLinearWithLoRA)
|
||||
layer._input_quant_key = object()
|
||||
assert get_input_quant_key(layer) is None
|
||||
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
not (
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def test_nemotron_h_lm_head_receives_quant_config():
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
|
||||
@@ -32,3 +34,28 @@ def test_nemotron_h_lm_head_receives_quant_config():
|
||||
MockLMHead.assert_called_once()
|
||||
call_kwargs = MockLMHead.call_args.kwargs
|
||||
assert call_kwargs["quant_config"] is mock_quant_config
|
||||
|
||||
|
||||
def test_relu2_fp8_fusion_uses_registry():
|
||||
from vllm.model_executor.models.nemotron_h import NemotronHMLP
|
||||
|
||||
projected = torch.empty((1, 1), dtype=torch.bfloat16)
|
||||
fused = Mock()
|
||||
act_fn = Mock()
|
||||
down_proj = Mock(side_effect=lambda x: (x, None))
|
||||
|
||||
mlp = NemotronHMLP.__new__(NemotronHMLP)
|
||||
torch.nn.Module.__init__(mlp)
|
||||
mlp.up_proj = Mock(return_value=(projected, None))
|
||||
mlp.down_proj = down_proj
|
||||
mlp.act_fn = act_fn
|
||||
|
||||
with patch(
|
||||
"vllm.model_executor.models.nemotron_h.maybe_fused_act_quant",
|
||||
return_value=fused,
|
||||
) as maybe_fused:
|
||||
result = mlp(Mock())
|
||||
|
||||
maybe_fused.assert_called_once_with(act_fn, projected, down_proj)
|
||||
act_fn.assert_not_called()
|
||||
assert result is fused
|
||||
|
||||
@@ -68,6 +68,10 @@ if envs.VLLM_LORA_ENABLE_DUAL_STREAM:
|
||||
|
||||
|
||||
class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
|
||||
# The adapter branch consumes the original activation, independently of
|
||||
# whether the wrapped base layer can consume a pre-quantized activation.
|
||||
requires_unquantized_input = True
|
||||
|
||||
def __init__(self, base_layer: LinearBase):
|
||||
super().__init__()
|
||||
|
||||
|
||||
@@ -5,21 +5,27 @@ Producer side of the QuantizedActivation contract for activation layers.
|
||||
|
||||
Given an activation module and the downstream linear it feeds, fuse the
|
||||
activation with that linear's input quantization into a single kernel when the
|
||||
linear advertises a consumable input_quant_key (see quant_activation.py).
|
||||
linear advertises a consumable input quantization key (see quant_activation.py).
|
||||
Falls back to the plain activation when nothing matches, so a model forward can
|
||||
always call maybe_fused_act_quant unconditionally.
|
||||
|
||||
This is the manual-fusion counterpart to ActivationQuantFusionPass: when fusion
|
||||
fires here the silu_and_mul pattern is already consumed, so the compiler pass
|
||||
finds nothing to rewrite and the two never double-fuse.
|
||||
fires here, the activation and quantization are already consumed, so a compiler
|
||||
pass cannot fuse the same boundary again.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.activation import ReLUSquaredActivation, SiluAndMul
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.relu2_fp8_quant import (
|
||||
relu_squared_static_fp8_quant,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -139,12 +145,46 @@ def _silu_and_mul_nvfp4_dynamic(
|
||||
)
|
||||
|
||||
|
||||
# (activation module type, consumer input_quant_key) -> fused producer.
|
||||
# Mirrors ActivationQuantFusionPass.FUSED_OPS; add a row to migrate a scheme.
|
||||
# (activation module type, consumer input quantization key) -> fused producer.
|
||||
# Add a row for each supported manual producer.
|
||||
_FUSED_ACT_QUANT: dict[tuple[type, QuantKey], Callable] = {
|
||||
(SiluAndMul, kFp8StaticTensorSym): _silu_and_mul_fp8_static,
|
||||
}
|
||||
|
||||
|
||||
def _relu_squared_static_fp8_quant_supported(
|
||||
act_fn: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
linear: LinearBase,
|
||||
) -> bool:
|
||||
"""Return whether the ReLU2 static-FP8 producer can consume this input."""
|
||||
scale = getattr(linear, "input_scale", None)
|
||||
return (
|
||||
isinstance(act_fn, ReLUSquaredActivation)
|
||||
and x.is_cuda
|
||||
and x.dtype == torch.bfloat16
|
||||
and x.is_contiguous()
|
||||
and isinstance(scale, torch.Tensor)
|
||||
and scale.dtype == torch.float32
|
||||
and scale.device == x.device
|
||||
and scale.numel() == 1
|
||||
)
|
||||
|
||||
|
||||
# Optional per-entry predicates preserve the unconditional fallback contract
|
||||
# for producers whose supported inputs are narrower than their registry key.
|
||||
_FUSED_ACT_QUANT_SUPPORT: dict[
|
||||
tuple[type, QuantKey], Callable[[torch.nn.Module, torch.Tensor, LinearBase], bool]
|
||||
] = {}
|
||||
|
||||
# This path is validated on SM90 and newer CUDA devices.
|
||||
if current_platform.is_cuda() and current_platform.has_device_capability(90):
|
||||
_RELU2_STATIC_FP8_KEY = (ReLUSquaredActivation, kFp8StaticTensorSym)
|
||||
_FUSED_ACT_QUANT[_RELU2_STATIC_FP8_KEY] = relu_squared_static_fp8_quant
|
||||
_FUSED_ACT_QUANT_SUPPORT[_RELU2_STATIC_FP8_KEY] = (
|
||||
_relu_squared_static_fp8_quant_supported
|
||||
)
|
||||
|
||||
# Add CUDA-specific entries for dynamic block quantization
|
||||
if current_platform.is_cuda_alike():
|
||||
_FUSED_ACT_QUANT[(SiluAndMul, kFp8Dynamic128Sym)] = _silu_and_mul_fp8_dynamic_128
|
||||
@@ -161,12 +201,14 @@ def maybe_fused_act_quant(
|
||||
) -> "torch.Tensor | QuantizedActivation":
|
||||
"""Apply act_fn, fusing the downstream linear's input quant when possible.
|
||||
|
||||
Returns a QuantizedActivation when a fused kernel matches
|
||||
(act_fn, linear.input_quant_key), else the plain activated tensor.
|
||||
Returns a QuantizedActivation when a fused kernel matches the activation and
|
||||
the consumer's effective input quantization key, else the plain activation.
|
||||
"""
|
||||
key = getattr(linear, "input_quant_key", None)
|
||||
key = get_input_quant_key(linear)
|
||||
if key is not None:
|
||||
producer = _FUSED_ACT_QUANT.get((type(act_fn), key))
|
||||
if producer is not None:
|
||||
registry_key = (type(act_fn), key)
|
||||
producer = _FUSED_ACT_QUANT.get(registry_key)
|
||||
support = _FUSED_ACT_QUANT_SUPPORT.get(registry_key)
|
||||
if producer is not None and (support is None or support(act_fn, x, linear)):
|
||||
return producer(x, linear)
|
||||
return act_fn(x)
|
||||
|
||||
@@ -5,7 +5,9 @@ A QuantizedActivation is a pre-quantized activation produced by a fused kernel
|
||||
and consumed directly by a linear layer, letting the layer skip its own input
|
||||
quantization. A linear advertises the key its kernel can consume via
|
||||
expose_input_quant_key; the kernel validates and reads the activation via
|
||||
as_quantized_activation.
|
||||
as_quantized_activation. Producers query the effective capability through
|
||||
get_input_quant_key, which hides the key when another consumer branch needs the
|
||||
original activation.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
@@ -46,12 +48,11 @@ class QuantizedActivation:
|
||||
|
||||
|
||||
def expose_input_quant_key(layer: torch.nn.Module, kernel) -> None:
|
||||
"""Advertise the kernel's pre-quantized input key on the layer, if any.
|
||||
"""Store the kernel's pre-quantized input key on the layer, if any.
|
||||
|
||||
This is the bridge from a kernel's input_quant_key() to the
|
||||
layer.input_quant_key attribute that fusion call sites read. The attribute
|
||||
is left unset when the kernel quantizes its own input, so non-supporting
|
||||
backends never receive a QuantizedActivation.
|
||||
This is the bridge from a kernel's input_quant_key() to the layer capability
|
||||
that fusion call sites read through get_input_quant_key. The raw key is left
|
||||
unset when the kernel quantizes its own input.
|
||||
|
||||
TODO(mgoin): Producers also need the consumer's quantization scales (e.g.
|
||||
static input scale, global scale). Expose those here as well so producers
|
||||
@@ -59,7 +60,14 @@ def expose_input_quant_key(layer: torch.nn.Module, kernel) -> None:
|
||||
"""
|
||||
key = kernel.input_quant_key()
|
||||
if key is not None:
|
||||
layer.input_quant_key = key
|
||||
layer._input_quant_key = key
|
||||
|
||||
|
||||
def get_input_quant_key(layer: torch.nn.Module) -> QuantKey | None:
|
||||
"""Return the pre-quantized input key the complete layer can consume."""
|
||||
if getattr(layer, "requires_unquantized_input", False):
|
||||
return None
|
||||
return getattr(layer, "_input_quant_key", None)
|
||||
|
||||
|
||||
def as_quantized_activation(
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_FP8_MIN, _FP8_MAX = get_fp8_min_max()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _relu_squared_static_fp8_quant_kernel(
|
||||
x_ptr,
|
||||
scale_ptr,
|
||||
output_ptr,
|
||||
num_elements,
|
||||
FP8_MIN: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < num_elements
|
||||
x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
|
||||
input_is_nan = x != x
|
||||
relu = tl.maximum(x, 0.0)
|
||||
# Preserve the standalone ReLU2 kernel's BF16 output boundary.
|
||||
activated = (relu * relu).to(tl.bfloat16).to(tl.float32)
|
||||
# Match O2's Inductor-compiled QuantFP8 reciprocal-then-multiply.
|
||||
quantized = activated * (1.0 / tl.load(scale_ptr))
|
||||
quantized = tl.maximum(tl.minimum(quantized, FP8_MAX), FP8_MIN)
|
||||
# Match the O2 native chain by preserving NaN through the FP8 conversion.
|
||||
quantized = tl.where(input_is_nan, x, quantized)
|
||||
tl.store(output_ptr + offsets, quantized, mask=mask)
|
||||
|
||||
|
||||
def relu_squared_static_fp8_quant(
|
||||
x: torch.Tensor, linear: LinearBase
|
||||
) -> QuantizedActivation:
|
||||
"""BF16 ReLU2 followed by static per-tensor FP8 quantization."""
|
||||
assert x.dtype == torch.bfloat16
|
||||
assert x.is_contiguous()
|
||||
scale = linear.input_scale
|
||||
assert scale.dtype == torch.float32
|
||||
assert x.device == scale.device
|
||||
assert scale.numel() == 1
|
||||
|
||||
output = torch.empty_like(x, dtype=current_platform.fp8_dtype())
|
||||
if x.numel() != 0:
|
||||
block_size = min(triton.next_power_of_2(x.shape[-1]), 2048)
|
||||
num_warps = min(max(block_size // 256, 1), 4)
|
||||
grid = lambda meta: (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),)
|
||||
_relu_squared_static_fp8_quant_kernel[grid](
|
||||
x,
|
||||
scale,
|
||||
output,
|
||||
x.numel(),
|
||||
FP8_MIN=_FP8_MIN,
|
||||
FP8_MAX=_FP8_MAX,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
|
||||
return QuantizedActivation(
|
||||
data=output,
|
||||
scale=scale,
|
||||
orig_dtype=x.dtype,
|
||||
orig_shape=x.shape,
|
||||
quant_key=kFp8StaticTensorSym,
|
||||
)
|
||||
@@ -252,6 +252,8 @@ class LinearBase(PluggableLayer):
|
||||
disable_tp: If true, tensor parallelism will be disabled for this layer.
|
||||
"""
|
||||
|
||||
requires_unquantized_input = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_size: int,
|
||||
|
||||
@@ -25,7 +25,11 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_gather
|
||||
@@ -38,6 +42,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
activation_without_mul,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -118,7 +123,7 @@ class NemotronHMLP(nn.Module):
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
x, _ = self.up_proj(x)
|
||||
x = self.act_fn(x)
|
||||
x = maybe_fused_act_quant(self.act_fn, x, self.down_proj)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
Reference in New Issue
Block a user