[Perf][Kernel][Quantization] Fuse ReLU2 with static FP8 activation quantization (#53793)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: ElizaWszola <ewszola@redhat.com>
Signed-off-by: snordmann <snordmann@nvidia.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: ElizaWszola <ewszola@redhat.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Samuel Nordmann
2026-09-14 10:26:15 -07:00
committed by GitHub
co-authored by mgoin Claude Opus 4.8 ElizaWszola OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 435c96f9db
commit 03dc26e639
12 changed files with 459 additions and 32 deletions
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.fusion.fused_act_quant import (
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
expose_input_quant_key,
get_input_quant_key,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
@@ -45,7 +46,7 @@ from vllm.platforms import current_platform
# Mock linear layer for testing fusion paths that don't have real kernel support
class MockLinearForFusion(torch.nn.Module):
"""Mock linear layer that exposes input_quant_key for fusion testing."""
"""Mock linear layer that exposes an input key for fusion testing."""
def __init__(
self,
@@ -55,7 +56,7 @@ class MockLinearForFusion(torch.nn.Module):
input_global_scale_inv=None,
):
super().__init__()
self.input_quant_key = quant_key
self._input_quant_key = quant_key
if input_scale is not None:
self.input_scale = input_scale
if input_global_scale is not None:
@@ -118,7 +119,7 @@ def test_manual_fusion_fp8_static_with_linear(
# Enable fusion
expose_input_quant_key(fp8_linear, fp8_linear.kernel)
if not hasattr(fp8_linear, "input_quant_key"):
if get_input_quant_key(fp8_linear) is None:
pytest.skip(
f"Kernel {force_kernel.__name__} doesn't support input_quant_key"
)
@@ -315,7 +316,7 @@ def test_manual_fusion_nvfp4_dynamic(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
"""Test that maybe_fused_act_quant falls back when no input_quant_key."""
"""Test that maybe_fused_act_quant falls back without an input key."""
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
@@ -327,7 +328,7 @@ def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
with set_current_vllm_config(config):
silu_and_mul = SiluAndMul()
# Linear without input_quant_key attribute
# Linear without an input quantization key
mock_linear = torch.nn.Linear(128, 128)
result = maybe_fused_act_quant(silu_and_mul, x, mock_linear)
@@ -42,6 +42,7 @@ from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
as_quantized_activation,
expose_input_quant_key,
get_input_quant_key,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
@@ -118,13 +119,13 @@ def test_bridge_marks_supporting_and_skips_others():
supported = _probe(FlashInferCutlassNvFp4LinearKernel)
layer = torch.nn.Module()
expose_input_quant_key(layer, supported)
assert layer.input_quant_key == kNvfp4Dynamic
assert get_input_quant_key(layer) == kNvfp4Dynamic
unsupported = _probe(FlashInferTrtllmNvFp4LinearKernel)
assert unsupported.input_quant_key() is None
layer = torch.nn.Module()
expose_input_quant_key(layer, unsupported)
assert not hasattr(layer, "input_quant_key")
assert get_input_quant_key(layer) is None
def test_as_quantized_activation_validates_key():
+26 -4
View File
@@ -91,7 +91,7 @@ class MockLinearFp8Static(torch.nn.Module):
def __init__(self, input_scale: torch.Tensor):
super().__init__()
self.input_quant_key = kFp8StaticTensorSym
self._input_quant_key = kFp8StaticTensorSym
self.input_scale = input_scale
@@ -100,15 +100,21 @@ class MockLinearFp8Dynamic128(torch.nn.Module):
def __init__(self):
super().__init__()
self.input_quant_key = kFp8Dynamic128Sym
self._input_quant_key = kFp8Dynamic128Sym
class MockLinearNoQuant(torch.nn.Module):
"""Mock linear layer with no input_quant_key (no fusion)."""
"""Mock linear layer with no input quantization key."""
pass
class MockLinearRequiresUnquantized(MockLinearFp8Static):
"""Mock consumer that needs the original activation for another branch."""
requires_unquantized_input = True
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
@pytest.mark.parametrize("dtype", DTYPES)
@@ -199,7 +205,7 @@ def test_maybe_fused_act_quant_fallback(
hidden_size: int,
dtype: torch.dtype,
) -> None:
"""Test maybe_fused_act_quant falls back when no input_quant_key."""
"""Test maybe_fused_act_quant falls back without an input quantization key."""
device = "cuda:0"
torch.set_default_device(device)
@@ -216,3 +222,19 @@ def test_maybe_fused_act_quant_fallback(
ref_out = act_fn(x)
torch.testing.assert_close(result, ref_out)
@torch.inference_mode()
def test_maybe_fused_act_quant_preserves_required_unquantized_input(
default_vllm_config,
) -> None:
device = "cuda:0"
act_fn = SiluAndMul()
scale = torch.tensor([0.5], device=device, dtype=torch.float32)
linear = MockLinearRequiresUnquantized(scale)
x = torch.randn(17, 256, dtype=torch.bfloat16, device=device)
result = maybe_fused_act_quant(act_fn, x, linear)
assert isinstance(result, torch.Tensor)
torch.testing.assert_close(result, act_fn(x))
+229
View File
@@ -0,0 +1,229 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
from vllm.config.vllm import OptimizationLevel
from vllm.model_executor.layers.activation import ReLUSquaredActivation
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
requires_sm90 = pytest.mark.skipif(
not (current_platform.is_cuda() and current_platform.has_device_capability(90)),
reason="This test requires SM90 or newer",
)
class _StaticFp8Linear(torch.nn.Module):
def __init__(self, input_scale: torch.Tensor) -> None:
super().__init__()
self._input_quant_key = kFp8StaticTensorSym
self.input_scale = input_scale
def _assert_fp8_bitwise_equal(actual: torch.Tensor, expected: torch.Tensor) -> None:
assert actual.dtype == expected.dtype == current_platform.fp8_dtype()
torch.testing.assert_close(
actual.view(torch.uint8), expected.view(torch.uint8), rtol=0, atol=0
)
def _assert_quantized_activation(
result: QuantizedActivation, x: torch.Tensor, scale: torch.Tensor
) -> None:
assert result.scale is scale
assert result.orig_dtype == x.dtype
assert result.orig_shape == x.shape
assert result.quant_key == kFp8StaticTensorSym
@pytest.fixture(scope="module")
def o2_relu2_fp8_ops():
config = VllmConfig(
optimization_level=OptimizationLevel.O2,
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
yield (
ReLUSquaredActivation(),
QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR),
)
@requires_sm90
@pytest.mark.parametrize("shape", [(1, 1), (1, 16), (17, 5120), (128, 10240)])
@torch.inference_mode()
def test_relu2_static_fp8_quant(o2_relu2_fp8_ops, shape) -> None:
torch.manual_seed(0)
x = torch.randn(shape, device="cuda", dtype=torch.bfloat16)
scale = torch.tensor(0.1495535671710968, device="cuda", dtype=torch.float32)
relu2, quant_fp8 = o2_relu2_fp8_ops
expected, _ = quant_fp8(relu2(x), scale)
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, QuantizedActivation)
_assert_quantized_activation(result, x, scale)
_assert_fp8_bitwise_equal(result.data, expected)
@requires_sm90
@pytest.mark.parametrize(
"enabled_ops",
[(), ("relu2",), ("quant_fp8",), ("relu2", "quant_fp8")],
)
@torch.inference_mode()
def test_relu2_static_fp8_quant_ignores_custom_op_dispatch(enabled_ops) -> None:
custom_ops = ["none", *(f"+{op}" for op in enabled_ops)]
config = VllmConfig(
optimization_level=OptimizationLevel.O2,
compilation_config=CompilationConfig(custom_ops=custom_ops),
)
with set_current_vllm_config(config):
torch.manual_seed(0)
relu2 = ReLUSquaredActivation()
quant_fp8 = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
expected, _ = quant_fp8(relu2(x), scale)
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, QuantizedActivation)
_assert_quantized_activation(result, x, scale)
_assert_fp8_bitwise_equal(result.data, expected)
@requires_sm90
@torch.inference_mode()
def test_relu2_static_fp8_quant_without_compilation() -> None:
config = VllmConfig(optimization_level=OptimizationLevel.O0)
with set_current_vllm_config(config):
torch.manual_seed(0)
relu2 = ReLUSquaredActivation()
quant_fp8 = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
expected, _ = quant_fp8(relu2(x), scale)
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, QuantizedActivation)
_assert_quantized_activation(result, x, scale)
_assert_fp8_bitwise_equal(result.data, expected)
@requires_sm90
@pytest.mark.parametrize(
"scale_value",
[
0.00435965,
0.007149832788854837,
0.03125,
0.0714285746216774,
0.1495535671710968,
0.1517857164144516,
6.03571415,
],
)
@torch.inference_mode()
def test_relu2_static_fp8_quant_all_bf16_values(o2_relu2_fp8_ops, scale_value) -> None:
bits = torch.arange(1 << 16, dtype=torch.int32).to(torch.uint16)
x = bits.view(torch.bfloat16).reshape(1, -1).to("cuda")
scale = torch.tensor(scale_value, device="cuda", dtype=torch.float32)
relu2, quant_fp8 = o2_relu2_fp8_ops
expected, _ = quant_fp8(relu2(x), scale)
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, QuantizedActivation)
_assert_quantized_activation(result, x, scale)
_assert_fp8_bitwise_equal(result.data, expected)
@requires_sm90
@torch.inference_mode()
def test_relu2_static_fp8_quant_empty(o2_relu2_fp8_ops) -> None:
x = torch.empty(0, device="cuda", dtype=torch.bfloat16)
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
relu2, _ = o2_relu2_fp8_ops
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, QuantizedActivation)
_assert_quantized_activation(result, x, scale)
assert result.data.shape == x.shape
assert result.data.dtype == current_platform.fp8_dtype()
@requires_sm90
@pytest.mark.parametrize("unsupported", ["dtype", "layout", "scale"])
@torch.inference_mode()
def test_relu2_static_fp8_quant_falls_back(o2_relu2_fp8_ops, unsupported: str) -> None:
x = torch.randn((17, 32), device="cuda", dtype=torch.bfloat16)
scale = torch.tensor(0.03125, device="cuda", dtype=torch.float32)
if unsupported == "dtype":
x = x.to(torch.float16)
elif unsupported == "layout":
x = x.T
else:
scale = scale.to(torch.float16)
relu2, _ = o2_relu2_fp8_ops
expected = relu2(x)
result = maybe_fused_act_quant(relu2, x, _StaticFp8Linear(scale))
assert isinstance(result, torch.Tensor)
assert not isinstance(result, QuantizedActivation)
torch.testing.assert_close(result, expected, rtol=0, atol=0)
@requires_sm90
@torch.inference_mode()
def test_relu2_static_fp8_quant_torch_compile_dynamic(
o2_relu2_fp8_ops,
) -> None:
torch.manual_seed(1)
scale = torch.tensor(0.00435965, device="cuda", dtype=torch.float32)
relu2, quant_fp8 = o2_relu2_fp8_ops
linear = _StaticFp8Linear(scale)
def fused(x: torch.Tensor) -> torch.Tensor:
result = maybe_fused_act_quant(relu2, x, linear)
assert isinstance(result, QuantizedActivation)
return result.data
compile_count = 0
def counting_inductor_backend(gm, example_inputs):
nonlocal compile_count
compile_count += 1
return torch._inductor.compile(gm, example_inputs)
torch._dynamo.reset()
try:
x = torch.randn((17, 5120), device="cuda", dtype=torch.bfloat16)
torch._dynamo.mark_dynamic(x, 0)
compiled_fused = torch.compile(
fused, backend=counting_inductor_backend, fullgraph=True
)
for candidate in (
x,
torch.randn((31, 5120), device="cuda", dtype=torch.bfloat16),
torch.randn((63, 5120), device="cuda", dtype=torch.bfloat16),
):
expected, _ = quant_fp8(relu2(candidate), scale)
_assert_fp8_bitwise_equal(compiled_fused(candidate), expected)
assert compile_count == 1
finally:
torch._dynamo.reset()
+10
View File
@@ -31,6 +31,9 @@ from vllm.lora.layers import (
)
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
from vllm.lora.punica_wrapper import get_punica_wrapper
from vllm.model_executor.layers.fusion.quant_activation import (
get_input_quant_key,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
@@ -56,6 +59,13 @@ TOLERANCES = {
torch.bfloat16: (3e-2, 2e-2),
}
def test_lora_linear_requires_unquantized_input() -> None:
layer = RowParallelLinearWithLoRA.__new__(RowParallelLinearWithLoRA)
layer._input_quant_key = object()
assert get_input_quant_key(layer) is None
pytestmark = [
pytest.mark.skipif(
not (
@@ -3,6 +3,8 @@
from unittest.mock import Mock, patch
import torch
def test_nemotron_h_lm_head_receives_quant_config():
from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
@@ -32,3 +34,28 @@ def test_nemotron_h_lm_head_receives_quant_config():
MockLMHead.assert_called_once()
call_kwargs = MockLMHead.call_args.kwargs
assert call_kwargs["quant_config"] is mock_quant_config
def test_relu2_fp8_fusion_uses_registry():
from vllm.model_executor.models.nemotron_h import NemotronHMLP
projected = torch.empty((1, 1), dtype=torch.bfloat16)
fused = Mock()
act_fn = Mock()
down_proj = Mock(side_effect=lambda x: (x, None))
mlp = NemotronHMLP.__new__(NemotronHMLP)
torch.nn.Module.__init__(mlp)
mlp.up_proj = Mock(return_value=(projected, None))
mlp.down_proj = down_proj
mlp.act_fn = act_fn
with patch(
"vllm.model_executor.models.nemotron_h.maybe_fused_act_quant",
return_value=fused,
) as maybe_fused:
result = mlp(Mock())
maybe_fused.assert_called_once_with(act_fn, projected, down_proj)
act_fn.assert_not_called()
assert result is fused
+4
View File
@@ -68,6 +68,10 @@ if envs.VLLM_LORA_ENABLE_DUAL_STREAM:
class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
# The adapter branch consumes the original activation, independently of
# whether the wrapped base layer can consume a pre-quantized activation.
requires_unquantized_input = True
def __init__(self, base_layer: LinearBase):
super().__init__()
@@ -5,21 +5,27 @@ Producer side of the QuantizedActivation contract for activation layers.
Given an activation module and the downstream linear it feeds, fuse the
activation with that linear's input quantization into a single kernel when the
linear advertises a consumable input_quant_key (see quant_activation.py).
linear advertises a consumable input quantization key (see quant_activation.py).
Falls back to the plain activation when nothing matches, so a model forward can
always call maybe_fused_act_quant unconditionally.
This is the manual-fusion counterpart to ActivationQuantFusionPass: when fusion
fires here the silu_and_mul pattern is already consumed, so the compiler pass
finds nothing to rewrite and the two never double-fuse.
fires here, the activation and quantization are already consumed, so a compiler
pass cannot fuse the same boundary again.
"""
from collections.abc import Callable
import torch
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.activation import ReLUSquaredActivation, SiluAndMul
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
get_input_quant_key,
)
from vllm.model_executor.layers.fusion.relu2_fp8_quant import (
relu_squared_static_fp8_quant,
)
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -139,12 +145,46 @@ def _silu_and_mul_nvfp4_dynamic(
)
# (activation module type, consumer input_quant_key) -> fused producer.
# Mirrors ActivationQuantFusionPass.FUSED_OPS; add a row to migrate a scheme.
# (activation module type, consumer input quantization key) -> fused producer.
# Add a row for each supported manual producer.
_FUSED_ACT_QUANT: dict[tuple[type, QuantKey], Callable] = {
(SiluAndMul, kFp8StaticTensorSym): _silu_and_mul_fp8_static,
}
def _relu_squared_static_fp8_quant_supported(
act_fn: torch.nn.Module,
x: torch.Tensor,
linear: LinearBase,
) -> bool:
"""Return whether the ReLU2 static-FP8 producer can consume this input."""
scale = getattr(linear, "input_scale", None)
return (
isinstance(act_fn, ReLUSquaredActivation)
and x.is_cuda
and x.dtype == torch.bfloat16
and x.is_contiguous()
and isinstance(scale, torch.Tensor)
and scale.dtype == torch.float32
and scale.device == x.device
and scale.numel() == 1
)
# Optional per-entry predicates preserve the unconditional fallback contract
# for producers whose supported inputs are narrower than their registry key.
_FUSED_ACT_QUANT_SUPPORT: dict[
tuple[type, QuantKey], Callable[[torch.nn.Module, torch.Tensor, LinearBase], bool]
] = {}
# This path is validated on SM90 and newer CUDA devices.
if current_platform.is_cuda() and current_platform.has_device_capability(90):
_RELU2_STATIC_FP8_KEY = (ReLUSquaredActivation, kFp8StaticTensorSym)
_FUSED_ACT_QUANT[_RELU2_STATIC_FP8_KEY] = relu_squared_static_fp8_quant
_FUSED_ACT_QUANT_SUPPORT[_RELU2_STATIC_FP8_KEY] = (
_relu_squared_static_fp8_quant_supported
)
# Add CUDA-specific entries for dynamic block quantization
if current_platform.is_cuda_alike():
_FUSED_ACT_QUANT[(SiluAndMul, kFp8Dynamic128Sym)] = _silu_and_mul_fp8_dynamic_128
@@ -161,12 +201,14 @@ def maybe_fused_act_quant(
) -> "torch.Tensor | QuantizedActivation":
"""Apply act_fn, fusing the downstream linear's input quant when possible.
Returns a QuantizedActivation when a fused kernel matches
(act_fn, linear.input_quant_key), else the plain activated tensor.
Returns a QuantizedActivation when a fused kernel matches the activation and
the consumer's effective input quantization key, else the plain activation.
"""
key = getattr(linear, "input_quant_key", None)
key = get_input_quant_key(linear)
if key is not None:
producer = _FUSED_ACT_QUANT.get((type(act_fn), key))
if producer is not None:
registry_key = (type(act_fn), key)
producer = _FUSED_ACT_QUANT.get(registry_key)
support = _FUSED_ACT_QUANT_SUPPORT.get(registry_key)
if producer is not None and (support is None or support(act_fn, x, linear)):
return producer(x, linear)
return act_fn(x)
@@ -5,7 +5,9 @@ A QuantizedActivation is a pre-quantized activation produced by a fused kernel
and consumed directly by a linear layer, letting the layer skip its own input
quantization. A linear advertises the key its kernel can consume via
expose_input_quant_key; the kernel validates and reads the activation via
as_quantized_activation.
as_quantized_activation. Producers query the effective capability through
get_input_quant_key, which hides the key when another consumer branch needs the
original activation.
"""
from dataclasses import dataclass, replace
@@ -46,12 +48,11 @@ class QuantizedActivation:
def expose_input_quant_key(layer: torch.nn.Module, kernel) -> None:
"""Advertise the kernel's pre-quantized input key on the layer, if any.
"""Store the kernel's pre-quantized input key on the layer, if any.
This is the bridge from a kernel's input_quant_key() to the
layer.input_quant_key attribute that fusion call sites read. The attribute
is left unset when the kernel quantizes its own input, so non-supporting
backends never receive a QuantizedActivation.
This is the bridge from a kernel's input_quant_key() to the layer capability
that fusion call sites read through get_input_quant_key. The raw key is left
unset when the kernel quantizes its own input.
TODO(mgoin): Producers also need the consumer's quantization scales (e.g.
static input scale, global scale). Expose those here as well so producers
@@ -59,7 +60,14 @@ def expose_input_quant_key(layer: torch.nn.Module, kernel) -> None:
"""
key = kernel.input_quant_key()
if key is not None:
layer.input_quant_key = key
layer._input_quant_key = key
def get_input_quant_key(layer: torch.nn.Module) -> QuantKey | None:
"""Return the pre-quantized input key the complete layer can consume."""
if getattr(layer, "requires_unquantized_input", False):
return None
return getattr(layer, "_input_quant_key", None)
def as_quantized_activation(
@@ -0,0 +1,76 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
_FP8_MIN, _FP8_MAX = get_fp8_min_max()
@triton.jit
def _relu_squared_static_fp8_quant_kernel(
x_ptr,
scale_ptr,
output_ptr,
num_elements,
FP8_MIN: tl.constexpr,
FP8_MAX: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < num_elements
x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
input_is_nan = x != x
relu = tl.maximum(x, 0.0)
# Preserve the standalone ReLU2 kernel's BF16 output boundary.
activated = (relu * relu).to(tl.bfloat16).to(tl.float32)
# Match O2's Inductor-compiled QuantFP8 reciprocal-then-multiply.
quantized = activated * (1.0 / tl.load(scale_ptr))
quantized = tl.maximum(tl.minimum(quantized, FP8_MAX), FP8_MIN)
# Match the O2 native chain by preserving NaN through the FP8 conversion.
quantized = tl.where(input_is_nan, x, quantized)
tl.store(output_ptr + offsets, quantized, mask=mask)
def relu_squared_static_fp8_quant(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""BF16 ReLU2 followed by static per-tensor FP8 quantization."""
assert x.dtype == torch.bfloat16
assert x.is_contiguous()
scale = linear.input_scale
assert scale.dtype == torch.float32
assert x.device == scale.device
assert scale.numel() == 1
output = torch.empty_like(x, dtype=current_platform.fp8_dtype())
if x.numel() != 0:
block_size = min(triton.next_power_of_2(x.shape[-1]), 2048)
num_warps = min(max(block_size // 256, 1), 4)
grid = lambda meta: (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),)
_relu_squared_static_fp8_quant_kernel[grid](
x,
scale,
output,
x.numel(),
FP8_MIN=_FP8_MIN,
FP8_MAX=_FP8_MAX,
BLOCK_SIZE=block_size,
num_warps=num_warps,
)
return QuantizedActivation(
data=output,
scale=scale,
orig_dtype=x.dtype,
orig_shape=x.shape,
quant_key=kFp8StaticTensorSym,
)
+2
View File
@@ -252,6 +252,8 @@ class LinearBase(PluggableLayer):
disable_tp: If true, tensor parallelism will be disabled for this layer.
"""
requires_unquantized_input = False
def __init__(
self,
input_size: int,
+7 -2
View File
@@ -25,7 +25,11 @@ import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.config import (
CacheConfig,
ModelConfig,
VllmConfig,
)
from vllm.config.parallel import ParallelConfig
from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size
from vllm.distributed.communication_op import tensor_model_parallel_all_gather
@@ -38,6 +42,7 @@ from vllm.model_executor.layers.fused_moe import (
activation_without_mul,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -118,7 +123,7 @@ class NemotronHMLP(nn.Module):
def forward(self, x: torch.Tensor):
x, _ = self.up_proj(x)
x = self.act_fn(x)
x = maybe_fused_act_quant(self.act_fn, x, self.down_proj)
x, _ = self.down_proj(x)
return x