From 8ebc5b0a182b3351a1c62fe73e97be2c489ed2a6 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Tue, 8 Sep 2026 23:26:07 +0800 Subject: [PATCH] [Bugfix] Fix NVFP4 fused SiLU+mul scale allocation and global scale direction (#55643) Signed-off-by: zjy0516 Co-authored-by: khluu Co-authored-by: jiahao Co-authored-by: Kimi Code --- .buildkite/test_areas/compile.yaml | 3 ++ .buildkite/test_areas/kernels.yaml | 4 ++ .../test_silu_mul_quant_manual_fusion.py | 47 +++++++++++++++---- .../quantization/test_silu_mul_nvfp4_quant.py | 2 +- .../layers/fusion/fused_act_quant.py | 29 ++++++++---- 5 files changed, 67 insertions(+), 18 deletions(-) diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index ac0112ad5e..8eea800686 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -46,6 +46,7 @@ steps: source_file_dependencies: - csrc/quantization/fp4/ - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fusion/ - vllm/model_executor/layers/layernorm.py - vllm/model_executor/layers/activation.py - vllm/model_executor/layers/attention/attention.py @@ -54,6 +55,7 @@ steps: - tests/compile/passes/test_fusion_attn.py - tests/compile/passes/test_mla_attn_quant_fusion.py - tests/compile/passes/test_silu_mul_quant_fusion.py + - tests/compile/passes/test_silu_mul_quant_manual_fusion.py - tests/compile/passes/distributed/test_fusion_all_reduce.py - tests/compile/fullgraph/test_full_graph.py commands: @@ -62,6 +64,7 @@ steps: - pytest -v -s tests/compile/passes/test_fusion_attn.py -k FLASHINFER - pytest -v -s tests/compile/passes/test_mla_attn_quant_fusion.py - pytest -v -s tests/compile/passes/test_silu_mul_quant_fusion.py + - pytest -v -s tests/compile/passes/test_silu_mul_quant_manual_fusion.py # this runner has 2 GPUs available even though num_devices=2 is not set - pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 12fe3dd277..4a19bc1795 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -312,8 +312,10 @@ steps: # optional: true source_file_dependencies: - csrc/quantization/fp4/ + - csrc/libtorch_stable/quantization/fp4/ - csrc/attention/mla/ - csrc/quantization/cutlass_w8a8/moe/ + - vllm/model_executor/layers/fusion/ - vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py - vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py - vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py @@ -349,6 +351,8 @@ steps: - tests/kernels/attention/test_minimax_m3.py - tests/kernels/test_ll_bf16_gemm.py - tests/kernels/test_top_k_per_row.py + - tests/kernels/quantization/test_silu_mul_nvfp4_quant.py + - tests/models/quantization/test_nvfp4.py commands: - nvidia-smi - | diff --git a/tests/compile/passes/test_silu_mul_quant_manual_fusion.py b/tests/compile/passes/test_silu_mul_quant_manual_fusion.py index e41b0877f6..5ee749659b 100644 --- a/tests/compile/passes/test_silu_mul_quant_manual_fusion.py +++ b/tests/compile/passes/test_silu_mul_quant_manual_fusion.py @@ -47,13 +47,21 @@ from vllm.platforms import current_platform class MockLinearForFusion(torch.nn.Module): """Mock linear layer that exposes input_quant_key for fusion testing.""" - def __init__(self, quant_key, input_scale=None, input_global_scale=None): + def __init__( + self, + quant_key, + input_scale=None, + input_global_scale=None, + input_global_scale_inv=None, + ): super().__init__() self.input_quant_key = quant_key if input_scale is not None: self.input_scale = input_scale if input_global_scale is not None: self.input_global_scale = input_global_scale + if input_global_scale_inv is not None: + self.input_global_scale_inv = input_global_scale_inv ROCM_KERNELS = [ROCmFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel] @@ -206,6 +214,10 @@ def test_manual_fusion_fp8_dynamic_128(dtype: torch.dtype): @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +# Cover M < 128 (padded swizzled SF tiles) and hidden_size that makes the +# kernel launch grid.y > 1 (num_packed_cols = hidden_size / 16 > 512). +@pytest.mark.parametrize("num_tokens", [1, 127, 128]) +@pytest.mark.parametrize("hidden_size", [256, 14336]) @pytest.mark.skipif(not current_platform.is_cuda(), reason="NVFP4 CUDA only") @pytest.mark.skipif( not current_platform.has_device_capability(100), reason="NVFP4 requires SM100+" @@ -213,7 +225,9 @@ def test_manual_fusion_fp8_dynamic_128(dtype: torch.dtype): @pytest.mark.skipif( envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm" ) -def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype): +def test_manual_fusion_nvfp4_dynamic( + dtype: torch.dtype, num_tokens: int, hidden_size: int +): """Test kNvfp4Dynamic fusion path. Compares fused (silu_and_mul_nvfp4_quant) vs unfused (silu_and_mul) @@ -228,10 +242,12 @@ def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype): torch.set_default_dtype(dtype) # NVFP4 requires hidden_size divisible by 16 (block size) and by 2 (packing). - # M (num_tokens) must be >= 128 for the 128x4 swizzled scale layout. - num_tokens, hidden_size = 128, 256 x = torch.rand(num_tokens, hidden_size * 2) - input_global_scale = torch.tensor([1.0], dtype=torch.float32, device="cuda") + # Non-1.0 global scale so the test is sensitive to the scale direction: + # the fused producer must quantize with input_global_scale_inv (the GEMM's + # alpha divides by input_global_scale), matching the unfused path. + input_global_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda") + input_global_scale_inv = 1.0 / input_global_scale config = VllmConfig( compilation_config=CompilationConfig(custom_ops=["none"]), @@ -247,7 +263,9 @@ def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype): # Fused path: apply silu_and_mul + NVFP4 quantization mock_linear_with_key = MockLinearForFusion( - kNvfp4Dynamic, input_global_scale=input_global_scale + kNvfp4Dynamic, + input_global_scale=input_global_scale, + input_global_scale_inv=input_global_scale_inv, ) result_fused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_with_key) @@ -260,11 +278,24 @@ def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype): assert result_fused.orig_dtype == dtype assert result_fused.orig_shape == (num_tokens, hidden_size) - # Dequantize fused result and compare with unfused + # The scale tensor must use the padded 128x4-tile swizzled layout, + # same as scaled_fp4_quant's allocator; the kernel writes a full + # 128-row tile even when M is smaller. + from vllm.utils.math_utils import round_up + + assert result_fused.scale.dtype == current_platform.fp8_dtype() + assert result_fused.scale.shape == ( + round_up(num_tokens, 128), + round_up(hidden_size // 16, 4), + ) + + # Dequantize fused result and compare with unfused. The block scales + # must have been folded with input_global_scale_inv, so dequantize + # with the same inverse scale to recover the activation. dequant_result = dequantize_nvfp4_to_dtype( tensor_fp4=result_fused.data, tensor_sf=result_fused.scale, - global_scale=input_global_scale, + global_scale=input_global_scale_inv, dtype=dtype, device="cuda", block_size=16, diff --git a/tests/kernels/quantization/test_silu_mul_nvfp4_quant.py b/tests/kernels/quantization/test_silu_mul_nvfp4_quant.py index dd6c6abacb..02ab4f009f 100644 --- a/tests/kernels/quantization/test_silu_mul_nvfp4_quant.py +++ b/tests/kernels/quantization/test_silu_mul_nvfp4_quant.py @@ -23,7 +23,7 @@ FP4_DTYPE = torch.uint8 FP8_DTYPE = current_platform.fp8_dtype() DTYPES = [torch.float16, torch.bfloat16] -SHAPES = [(128, 256), (128, 128), (256, 256), (256, 128)] +SHAPES = [(128, 256), (128, 128), (256, 256), (256, 128), (1, 256), (64, 256)] BLOCK_SIZE = 16 diff --git a/vllm/model_executor/layers/fusion/fused_act_quant.py b/vllm/model_executor/layers/fusion/fused_act_quant.py index 534f94406a..9fe253f2d9 100644 --- a/vllm/model_executor/layers/fusion/fused_act_quant.py +++ b/vllm/model_executor/layers/fusion/fused_act_quant.py @@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kNvfp4Dynamic, ) from vllm.platforms import current_platform +from vllm.utils.math_utils import round_up FP8_DTYPE = current_platform.fp8_dtype() FP4_DTYPE = torch.uint8 @@ -105,20 +106,30 @@ def _silu_and_mul_nvfp4_dynamic( # NVFP4 packs 2 values into 1 byte result = torch.empty((num_tokens, d // 2), dtype=FP4_DTYPE, device=x.device) - # Block scale output shape: swizzled layout for tensor cores - # Each group of 16 elements shares one FP8 scale - num_k_tiles = (d + 63) // 64 + # Block scale output: swizzled tensor-core layout + # [num_m_tiles, num_k_tiles, 32, 4, 4] of int32-packed FP8 scales, so the + # row/col extents must be padded to the 128x4 tile like + # scaled_fp4_quant's allocator (create_fp4_scale_tensor). Each group of 16 + # elements shares one FP8 scale. + rounded_m = round_up(num_tokens, 128) + rounded_n = round_up(d // 16, 4) block_scale = torch.empty( - (num_tokens, num_k_tiles * 4), dtype=FP8_DTYPE, device=x.device + (rounded_m, rounded_n // 4), dtype=torch.int32, device=x.device + ).view(FP8_DTYPE) + + # The kernel folds the global scale into the block scales, and the + # consumer GEMM's alpha (= input_global_scale * weight_global_scale) + # divides it back out, so quantize with the reciprocal like the unfused + # scaled_fp4_quant path does. + input_global_scale_inv = getattr(linear, "input_global_scale_inv", None) + assert input_global_scale_inv is not None, ( + "input_global_scale_inv is required for NVFP4 quantization" ) - input_global_scale = getattr(linear, "input_global_scale", None) - assert input_global_scale is not None, ( - "input_global_scale is required for NVFP4 quantization" + torch.ops._C.silu_and_mul_nvfp4_quant( + result, block_scale, x, input_global_scale_inv ) - torch.ops._C.silu_and_mul_nvfp4_quant(result, block_scale, x, input_global_scale) - return QuantizedActivation( data=result.view(out_shape[:-1] + (d // 2,)), scale=block_scale,