[Bugfix] Fix NVFP4 fused SiLU+mul scale allocation and global scale direction (#55643)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: khluu <khluu000@gmail.com> Co-authored-by: jiahao <jxia77@terpmail.umd.edu> Co-authored-by: Kimi Code <noreply@moonshot.cn>
This commit is contained in:
co-authored by
khluu
jiahao
Kimi Code
parent
6b15bea080
commit
8ebc5b0a18
@@ -46,6 +46,7 @@ steps:
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/fp4/
|
||||
- vllm/model_executor/layers/quantization/
|
||||
- vllm/model_executor/layers/fusion/
|
||||
- vllm/model_executor/layers/layernorm.py
|
||||
- vllm/model_executor/layers/activation.py
|
||||
- vllm/model_executor/layers/attention/attention.py
|
||||
@@ -54,6 +55,7 @@ steps:
|
||||
- tests/compile/passes/test_fusion_attn.py
|
||||
- tests/compile/passes/test_mla_attn_quant_fusion.py
|
||||
- tests/compile/passes/test_silu_mul_quant_fusion.py
|
||||
- tests/compile/passes/test_silu_mul_quant_manual_fusion.py
|
||||
- tests/compile/passes/distributed/test_fusion_all_reduce.py
|
||||
- tests/compile/fullgraph/test_full_graph.py
|
||||
commands:
|
||||
@@ -62,6 +64,7 @@ steps:
|
||||
- pytest -v -s tests/compile/passes/test_fusion_attn.py -k FLASHINFER
|
||||
- pytest -v -s tests/compile/passes/test_mla_attn_quant_fusion.py
|
||||
- pytest -v -s tests/compile/passes/test_silu_mul_quant_fusion.py
|
||||
- pytest -v -s tests/compile/passes/test_silu_mul_quant_manual_fusion.py
|
||||
# this runner has 2 GPUs available even though num_devices=2 is not set
|
||||
- pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py
|
||||
|
||||
|
||||
@@ -312,8 +312,10 @@ steps:
|
||||
# optional: true
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/fp4/
|
||||
- csrc/libtorch_stable/quantization/fp4/
|
||||
- csrc/attention/mla/
|
||||
- csrc/quantization/cutlass_w8a8/moe/
|
||||
- vllm/model_executor/layers/fusion/
|
||||
- vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py
|
||||
@@ -349,6 +351,8 @@ steps:
|
||||
- tests/kernels/attention/test_minimax_m3.py
|
||||
- tests/kernels/test_ll_bf16_gemm.py
|
||||
- tests/kernels/test_top_k_per_row.py
|
||||
- tests/kernels/quantization/test_silu_mul_nvfp4_quant.py
|
||||
- tests/models/quantization/test_nvfp4.py
|
||||
commands:
|
||||
- nvidia-smi
|
||||
- |
|
||||
|
||||
@@ -47,13 +47,21 @@ from vllm.platforms import current_platform
|
||||
class MockLinearForFusion(torch.nn.Module):
|
||||
"""Mock linear layer that exposes input_quant_key for fusion testing."""
|
||||
|
||||
def __init__(self, quant_key, input_scale=None, input_global_scale=None):
|
||||
def __init__(
|
||||
self,
|
||||
quant_key,
|
||||
input_scale=None,
|
||||
input_global_scale=None,
|
||||
input_global_scale_inv=None,
|
||||
):
|
||||
super().__init__()
|
||||
self.input_quant_key = quant_key
|
||||
if input_scale is not None:
|
||||
self.input_scale = input_scale
|
||||
if input_global_scale is not None:
|
||||
self.input_global_scale = input_global_scale
|
||||
if input_global_scale_inv is not None:
|
||||
self.input_global_scale_inv = input_global_scale_inv
|
||||
|
||||
|
||||
ROCM_KERNELS = [ROCmFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel]
|
||||
@@ -206,6 +214,10 @@ def test_manual_fusion_fp8_dynamic_128(dtype: torch.dtype):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
# Cover M < 128 (padded swizzled SF tiles) and hidden_size that makes the
|
||||
# kernel launch grid.y > 1 (num_packed_cols = hidden_size / 16 > 512).
|
||||
@pytest.mark.parametrize("num_tokens", [1, 127, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [256, 14336])
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="NVFP4 CUDA only")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(100), reason="NVFP4 requires SM100+"
|
||||
@@ -213,7 +225,9 @@ def test_manual_fusion_fp8_dynamic_128(dtype: torch.dtype):
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
|
||||
)
|
||||
def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype):
|
||||
def test_manual_fusion_nvfp4_dynamic(
|
||||
dtype: torch.dtype, num_tokens: int, hidden_size: int
|
||||
):
|
||||
"""Test kNvfp4Dynamic fusion path.
|
||||
|
||||
Compares fused (silu_and_mul_nvfp4_quant) vs unfused (silu_and_mul)
|
||||
@@ -228,10 +242,12 @@ def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype):
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
# NVFP4 requires hidden_size divisible by 16 (block size) and by 2 (packing).
|
||||
# M (num_tokens) must be >= 128 for the 128x4 swizzled scale layout.
|
||||
num_tokens, hidden_size = 128, 256
|
||||
x = torch.rand(num_tokens, hidden_size * 2)
|
||||
input_global_scale = torch.tensor([1.0], dtype=torch.float32, device="cuda")
|
||||
# Non-1.0 global scale so the test is sensitive to the scale direction:
|
||||
# the fused producer must quantize with input_global_scale_inv (the GEMM's
|
||||
# alpha divides by input_global_scale), matching the unfused path.
|
||||
input_global_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
input_global_scale_inv = 1.0 / input_global_scale
|
||||
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=["none"]),
|
||||
@@ -247,7 +263,9 @@ def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype):
|
||||
|
||||
# Fused path: apply silu_and_mul + NVFP4 quantization
|
||||
mock_linear_with_key = MockLinearForFusion(
|
||||
kNvfp4Dynamic, input_global_scale=input_global_scale
|
||||
kNvfp4Dynamic,
|
||||
input_global_scale=input_global_scale,
|
||||
input_global_scale_inv=input_global_scale_inv,
|
||||
)
|
||||
result_fused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_with_key)
|
||||
|
||||
@@ -260,11 +278,24 @@ def test_manual_fusion_nvfp4_dynamic(dtype: torch.dtype):
|
||||
assert result_fused.orig_dtype == dtype
|
||||
assert result_fused.orig_shape == (num_tokens, hidden_size)
|
||||
|
||||
# Dequantize fused result and compare with unfused
|
||||
# The scale tensor must use the padded 128x4-tile swizzled layout,
|
||||
# same as scaled_fp4_quant's allocator; the kernel writes a full
|
||||
# 128-row tile even when M is smaller.
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
assert result_fused.scale.dtype == current_platform.fp8_dtype()
|
||||
assert result_fused.scale.shape == (
|
||||
round_up(num_tokens, 128),
|
||||
round_up(hidden_size // 16, 4),
|
||||
)
|
||||
|
||||
# Dequantize fused result and compare with unfused. The block scales
|
||||
# must have been folded with input_global_scale_inv, so dequantize
|
||||
# with the same inverse scale to recover the activation.
|
||||
dequant_result = dequantize_nvfp4_to_dtype(
|
||||
tensor_fp4=result_fused.data,
|
||||
tensor_sf=result_fused.scale,
|
||||
global_scale=input_global_scale,
|
||||
global_scale=input_global_scale_inv,
|
||||
dtype=dtype,
|
||||
device="cuda",
|
||||
block_size=16,
|
||||
|
||||
@@ -23,7 +23,7 @@ FP4_DTYPE = torch.uint8
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
SHAPES = [(128, 256), (128, 128), (256, 256), (256, 128)]
|
||||
SHAPES = [(128, 256), (128, 128), (256, 256), (256, 128), (1, 256), (64, 256)]
|
||||
BLOCK_SIZE = 16
|
||||
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP4_DTYPE = torch.uint8
|
||||
@@ -105,20 +106,30 @@ def _silu_and_mul_nvfp4_dynamic(
|
||||
# NVFP4 packs 2 values into 1 byte
|
||||
result = torch.empty((num_tokens, d // 2), dtype=FP4_DTYPE, device=x.device)
|
||||
|
||||
# Block scale output shape: swizzled layout for tensor cores
|
||||
# Each group of 16 elements shares one FP8 scale
|
||||
num_k_tiles = (d + 63) // 64
|
||||
# Block scale output: swizzled tensor-core layout
|
||||
# [num_m_tiles, num_k_tiles, 32, 4, 4] of int32-packed FP8 scales, so the
|
||||
# row/col extents must be padded to the 128x4 tile like
|
||||
# scaled_fp4_quant's allocator (create_fp4_scale_tensor). Each group of 16
|
||||
# elements shares one FP8 scale.
|
||||
rounded_m = round_up(num_tokens, 128)
|
||||
rounded_n = round_up(d // 16, 4)
|
||||
block_scale = torch.empty(
|
||||
(num_tokens, num_k_tiles * 4), dtype=FP8_DTYPE, device=x.device
|
||||
(rounded_m, rounded_n // 4), dtype=torch.int32, device=x.device
|
||||
).view(FP8_DTYPE)
|
||||
|
||||
# The kernel folds the global scale into the block scales, and the
|
||||
# consumer GEMM's alpha (= input_global_scale * weight_global_scale)
|
||||
# divides it back out, so quantize with the reciprocal like the unfused
|
||||
# scaled_fp4_quant path does.
|
||||
input_global_scale_inv = getattr(linear, "input_global_scale_inv", None)
|
||||
assert input_global_scale_inv is not None, (
|
||||
"input_global_scale_inv is required for NVFP4 quantization"
|
||||
)
|
||||
|
||||
input_global_scale = getattr(linear, "input_global_scale", None)
|
||||
assert input_global_scale is not None, (
|
||||
"input_global_scale is required for NVFP4 quantization"
|
||||
torch.ops._C.silu_and_mul_nvfp4_quant(
|
||||
result, block_scale, x, input_global_scale_inv
|
||||
)
|
||||
|
||||
torch.ops._C.silu_and_mul_nvfp4_quant(result, block_scale, x, input_global_scale)
|
||||
|
||||
return QuantizedActivation(
|
||||
data=result.view(out_shape[:-1] + (d // 2,)),
|
||||
scale=block_scale,
|
||||
|
||||
Reference in New Issue
Block a user