[Bugfix] Fix NVFP4 fused SiLU+mul scale allocation and global scale direction (#55643)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: khluu <khluu000@gmail.com>
Co-authored-by: jiahao <jxia77@terpmail.umd.edu>
Co-authored-by: Kimi Code <noreply@moonshot.cn>
This commit is contained in:
Jiangyun Zhu
2026-09-08 08:26:07 -07:00
committed by GitHub
co-authored by khluu jiahao Kimi Code
parent 6b15bea080
commit 8ebc5b0a18
5 changed files with 67 additions and 18 deletions
+3
View File
@@ -46,6 +46,7 @@ steps:
source_file_dependencies:
- csrc/quantization/fp4/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fusion/
- vllm/model_executor/layers/layernorm.py
- vllm/model_executor/layers/activation.py
- vllm/model_executor/layers/attention/attention.py
@@ -54,6 +55,7 @@ steps:
- tests/compile/passes/test_fusion_attn.py
- tests/compile/passes/test_mla_attn_quant_fusion.py
- tests/compile/passes/test_silu_mul_quant_fusion.py
- tests/compile/passes/test_silu_mul_quant_manual_fusion.py
- tests/compile/passes/distributed/test_fusion_all_reduce.py
- tests/compile/fullgraph/test_full_graph.py
commands:
@@ -62,6 +64,7 @@ steps:
- pytest -v -s tests/compile/passes/test_fusion_attn.py -k FLASHINFER
- pytest -v -s tests/compile/passes/test_mla_attn_quant_fusion.py
- pytest -v -s tests/compile/passes/test_silu_mul_quant_fusion.py
- pytest -v -s tests/compile/passes/test_silu_mul_quant_manual_fusion.py
# this runner has 2 GPUs available even though num_devices=2 is not set
- pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py
+4
View File
@@ -312,8 +312,10 @@ steps:
# optional: true
source_file_dependencies:
- csrc/quantization/fp4/
- csrc/libtorch_stable/quantization/fp4/
- csrc/attention/mla/
- csrc/quantization/cutlass_w8a8/moe/
- vllm/model_executor/layers/fusion/
- vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py
- vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py
- vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py
@@ -349,6 +351,8 @@ steps:
- tests/kernels/attention/test_minimax_m3.py
- tests/kernels/test_ll_bf16_gemm.py
- tests/kernels/test_top_k_per_row.py
- tests/kernels/quantization/test_silu_mul_nvfp4_quant.py
- tests/models/quantization/test_nvfp4.py
commands:
- nvidia-smi
- |
@@ -47,13 +47,21 @@ from vllm.platforms import current_platform
class MockLinearForFusion(torch.nn.Module):
"""Mock linear layer that exposes input_quant_key for fusion testing."""
def __init__(self, quant_key, input_scale=None, input_global_scale=None):
def __init__(
self,
quant_key,
input_scale=None,
input_global_scale=None,
input_global_scale_inv=None,
):
super().__init__()
self.input_quant_key = quant_key
if input_scale is not None:
self.input_scale = input_scale
if input_global_scale is not None:
self.input_global_scale = input_global_scale
if input_global_scale_inv is not None:
self.input_global_scale_inv = input_global_scale_inv
ROCM_KERNELS = [ROCmFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel]
@@ -206,6 +214,10 @@ def test_manual_fusion_fp8_dynamic_128(dtype: torch.dtype):
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
# Cover M < 128 (padded swizzled SF tiles) and hidden_size that makes the
# kernel launch grid.y > 1 (num_packed_cols = hidden_size / 16 > 512).
@pytest.mark.parametrize("num_tokens", [1, 127, 128])
@pytest.mark.parametrize("hidden_size", [256, 14336])
@pytest.mark.skipif(not current_platform.is_cuda(), reason="NVFP4 CUDA only")
@pytest.mark.skipif(
not current_platform.has_device_capability(100), reason="NVFP4 requires SM100+"
@@ -213,7 +225,9 @@ def test_manual_fusion_fp8_dynamic_128(dtype: torch.dtype):
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype):
def test_manual_fusion_nvfp4_dynamic(
dtype: torch.dtype, num_tokens: int, hidden_size: int
):
"""Test kNvfp4Dynamic fusion path.
Compares fused (silu_and_mul_nvfp4_quant) vs unfused (silu_and_mul)
@@ -228,10 +242,12 @@ def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype):
torch.set_default_dtype(dtype)
# NVFP4 requires hidden_size divisible by 16 (block size) and by 2 (packing).
# M (num_tokens) must be >= 128 for the 128x4 swizzled scale layout.
num_tokens, hidden_size = 128, 256
x = torch.rand(num_tokens, hidden_size * 2)
input_global_scale = torch.tensor([1.0], dtype=torch.float32, device="cuda")
# Non-1.0 global scale so the test is sensitive to the scale direction:
# the fused producer must quantize with input_global_scale_inv (the GEMM's
# alpha divides by input_global_scale), matching the unfused path.
input_global_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
input_global_scale_inv = 1.0 / input_global_scale
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
@@ -247,7 +263,9 @@ def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype):
# Fused path: apply silu_and_mul + NVFP4 quantization
mock_linear_with_key = MockLinearForFusion(
kNvfp4Dynamic, input_global_scale=input_global_scale
kNvfp4Dynamic,
input_global_scale=input_global_scale,
input_global_scale_inv=input_global_scale_inv,
)
result_fused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_with_key)
@@ -260,11 +278,24 @@ def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype):
assert result_fused.orig_dtype == dtype
assert result_fused.orig_shape == (num_tokens, hidden_size)
# Dequantize fused result and compare with unfused
# The scale tensor must use the padded 128x4-tile swizzled layout,
# same as scaled_fp4_quant's allocator; the kernel writes a full
# 128-row tile even when M is smaller.
from vllm.utils.math_utils import round_up
assert result_fused.scale.dtype == current_platform.fp8_dtype()
assert result_fused.scale.shape == (
round_up(num_tokens, 128),
round_up(hidden_size // 16, 4),
)
# Dequantize fused result and compare with unfused. The block scales
# must have been folded with input_global_scale_inv, so dequantize
# with the same inverse scale to recover the activation.
dequant_result = dequantize_nvfp4_to_dtype(
tensor_fp4=result_fused.data,
tensor_sf=result_fused.scale,
global_scale=input_global_scale,
global_scale=input_global_scale_inv,
dtype=dtype,
device="cuda",
block_size=16,
@@ -23,7 +23,7 @@ FP4_DTYPE = torch.uint8
FP8_DTYPE = current_platform.fp8_dtype()
DTYPES = [torch.float16, torch.bfloat16]
SHAPES = [(128, 256), (128, 128), (256, 256), (256, 128)]
SHAPES = [(128, 256), (128, 128), (256, 256), (256, 128), (1, 256), (64, 256)]
BLOCK_SIZE = 16
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
@@ -105,20 +106,30 @@ def _silu_and_mul_nvfp4_dynamic(
# NVFP4 packs 2 values into 1 byte
result = torch.empty((num_tokens, d // 2), dtype=FP4_DTYPE, device=x.device)
# Block scale output shape: swizzled layout for tensor cores
# Each group of 16 elements shares one FP8 scale
num_k_tiles = (d + 63) // 64
# Block scale output: swizzled tensor-core layout
# [num_m_tiles, num_k_tiles, 32, 4, 4] of int32-packed FP8 scales, so the
# row/col extents must be padded to the 128x4 tile like
# scaled_fp4_quant's allocator (create_fp4_scale_tensor). Each group of 16
# elements shares one FP8 scale.
rounded_m = round_up(num_tokens, 128)
rounded_n = round_up(d // 16, 4)
block_scale = torch.empty(
(num_tokens, num_k_tiles * 4), dtype=FP8_DTYPE, device=x.device
(rounded_m, rounded_n // 4), dtype=torch.int32, device=x.device
).view(FP8_DTYPE)
# The kernel folds the global scale into the block scales, and the
# consumer GEMM's alpha (= input_global_scale * weight_global_scale)
# divides it back out, so quantize with the reciprocal like the unfused
# scaled_fp4_quant path does.
input_global_scale_inv = getattr(linear, "input_global_scale_inv", None)
assert input_global_scale_inv is not None, (
"input_global_scale_inv is required for NVFP4 quantization"
)
input_global_scale = getattr(linear, "input_global_scale", None)
assert input_global_scale is not None, (
"input_global_scale is required for NVFP4 quantization"
torch.ops._C.silu_and_mul_nvfp4_quant(
result, block_scale, x, input_global_scale_inv
)
torch.ops._C.silu_and_mul_nvfp4_quant(result, block_scale, x, input_global_scale)
return QuantizedActivation(
data=result.view(out_shape[:-1] + (d // 2,)),
scale=block_scale,