[NVFP4][Hopper/AMD Instinct] Add Triton kernels for NVFP4 dequantization and QDQ emulation (#40033)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,308 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import huggingface_hub
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils import (
|
||||
nvfp4_emulation_utils,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_to_dtype,
|
||||
ref_nvfp4_quant_dequant,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Triton NVFP4 kernel requires CUDA.",
|
||||
)
|
||||
def test_triton_dequantize_nvfp4(monkeypatch) -> None:
|
||||
"""Test the Triton dequantization kernel against the CPU reference
|
||||
using real NVFP4 weights from a checkpoint.
|
||||
|
||||
Tests both 2D (attention projection) and 3D (stacked MoE experts).
|
||||
"""
|
||||
checkpoint_path = huggingface_hub.snapshot_download(
|
||||
"nvidia/Qwen3-30B-A3B-NVFP4",
|
||||
allow_patterns=["model-00001-of-00004.safetensors"],
|
||||
)
|
||||
shard_path = f"{checkpoint_path}/model-00001-of-00004.safetensors"
|
||||
block_size = 16
|
||||
|
||||
with safe_open(shard_path, framework="pt", device="cpu") as f:
|
||||
all_keys = list(f.keys())
|
||||
|
||||
# 2D case: attention projection
|
||||
tensor_fp4_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight")
|
||||
tensor_sf_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight_scale")
|
||||
global_scale_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight_scale_2")
|
||||
|
||||
# 3D case: stack ALL experts for layer 9 up_proj
|
||||
expert_prefix = "model.layers.9.mlp.experts."
|
||||
expert_indices = sorted(
|
||||
int(key.split(".")[5])
|
||||
for key in all_keys
|
||||
if key.startswith(expert_prefix) and key.endswith(".up_proj.weight")
|
||||
)
|
||||
assert len(expert_indices) > 0
|
||||
|
||||
all_fp4 = []
|
||||
all_sf = []
|
||||
all_global_scale = []
|
||||
for index in expert_indices:
|
||||
name = f"{expert_prefix}{index}.up_proj"
|
||||
all_fp4.append(f.get_tensor(f"{name}.weight"))
|
||||
all_sf.append(f.get_tensor(f"{name}.weight_scale"))
|
||||
all_global_scale.append(f.get_tensor(f"{name}.weight_scale_2"))
|
||||
|
||||
tensor_fp4_3d = torch.stack(all_fp4)
|
||||
tensor_sf_3d = torch.stack(all_sf)
|
||||
global_scale_3d = torch.stack(all_global_scale)
|
||||
|
||||
test_cases = [
|
||||
("2D base", tensor_fp4_2d, tensor_sf_2d, global_scale_2d),
|
||||
(
|
||||
"2D 2x rows",
|
||||
tensor_fp4_2d.repeat(2, 1),
|
||||
tensor_sf_2d.repeat(2, 1),
|
||||
global_scale_2d,
|
||||
),
|
||||
(
|
||||
"2D 4x rows",
|
||||
tensor_fp4_2d.repeat(4, 1),
|
||||
tensor_sf_2d.repeat(4, 1),
|
||||
global_scale_2d,
|
||||
),
|
||||
(
|
||||
"2D 2x cols",
|
||||
tensor_fp4_2d.repeat(1, 2),
|
||||
tensor_sf_2d.repeat(1, 2),
|
||||
global_scale_2d,
|
||||
),
|
||||
("3D base", tensor_fp4_3d, tensor_sf_3d, global_scale_3d),
|
||||
(
|
||||
"3D 2x experts",
|
||||
tensor_fp4_3d.repeat(2, 1, 1),
|
||||
tensor_sf_3d.repeat(2, 1, 1),
|
||||
global_scale_3d.repeat(2),
|
||||
),
|
||||
(
|
||||
"3D 2x rows",
|
||||
tensor_fp4_3d.repeat(1, 2, 1),
|
||||
tensor_sf_3d.repeat(1, 2, 1),
|
||||
global_scale_3d,
|
||||
),
|
||||
(
|
||||
"3D 2x cols",
|
||||
tensor_fp4_3d.repeat(1, 1, 2),
|
||||
tensor_sf_3d.repeat(1, 1, 2),
|
||||
global_scale_3d,
|
||||
),
|
||||
]
|
||||
|
||||
quantiles = [0.5, 0.001, 0.999]
|
||||
|
||||
# Move the E2M1 lookup table to CUDA ahead of time, as would normally
|
||||
# happen during model loading (process_weights_after_loading). Both the
|
||||
# Triton and PyTorch reference paths run on CUDA.
|
||||
nvfp4_emulation_utils.kE2M1ToFloat_handle.val = (
|
||||
nvfp4_emulation_utils.kE2M1ToFloat_handle.val.cuda()
|
||||
)
|
||||
|
||||
for label, tensor_fp4, tensor_sf, global_scale in test_cases:
|
||||
fp4_cuda = tensor_fp4.cuda()
|
||||
sf_cuda = tensor_sf.cuda()
|
||||
gs_cuda = global_scale.cuda()
|
||||
|
||||
# Triton path
|
||||
triton_result = dequantize_to_dtype(
|
||||
fp4_cuda,
|
||||
sf_cuda,
|
||||
gs_cuda,
|
||||
torch.bfloat16,
|
||||
block_size,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
# Reference path (PyTorch ops on CUDA, Triton dispatch disabled)
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(
|
||||
nvfp4_emulation_utils.current_platform,
|
||||
"is_cuda_alike",
|
||||
lambda: False,
|
||||
)
|
||||
reference = dequantize_to_dtype(
|
||||
fp4_cuda,
|
||||
sf_cuda,
|
||||
gs_cuda,
|
||||
torch.bfloat16,
|
||||
block_size,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(triton_result, reference, atol=0, rtol=0)
|
||||
|
||||
# Benchmark
|
||||
shape = list(tensor_fp4.shape)
|
||||
|
||||
def _triton_bench(
|
||||
fp4_cuda=fp4_cuda,
|
||||
scale_cuda=sf_cuda,
|
||||
global_scale_cuda=gs_cuda,
|
||||
block_size=block_size,
|
||||
):
|
||||
return dequantize_to_dtype(
|
||||
fp4_cuda,
|
||||
scale_cuda,
|
||||
global_scale_cuda,
|
||||
torch.bfloat16,
|
||||
block_size,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
triton_ms, triton_min, triton_max = triton.testing.do_bench(
|
||||
_triton_bench, quantiles=quantiles
|
||||
)
|
||||
|
||||
def _reference_bench(
|
||||
fp4_cuda=fp4_cuda,
|
||||
scale_cuda=sf_cuda,
|
||||
global_scale_cuda=gs_cuda,
|
||||
block_size=block_size,
|
||||
):
|
||||
with monkeypatch.context() as m2:
|
||||
m2.setattr(
|
||||
nvfp4_emulation_utils.current_platform,
|
||||
"is_cuda_alike",
|
||||
lambda: False,
|
||||
)
|
||||
dequantize_to_dtype(
|
||||
fp4_cuda,
|
||||
scale_cuda,
|
||||
global_scale_cuda,
|
||||
torch.bfloat16,
|
||||
block_size,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
ref_ms, ref_min, ref_max = triton.testing.do_bench(
|
||||
_reference_bench, quantiles=quantiles
|
||||
)
|
||||
|
||||
speedup = ref_ms / triton_ms if triton_ms > 0 else float("inf")
|
||||
print(f" dequantize {label} {shape}:")
|
||||
print(
|
||||
f" triton: median={triton_ms:.3f}ms, "
|
||||
f"min={triton_min:.3f}ms, max={triton_max:.3f}ms"
|
||||
)
|
||||
print(
|
||||
f" reference: median={ref_ms:.3f}ms, "
|
||||
f"min={ref_min:.3f}ms, max={ref_max:.3f}ms"
|
||||
)
|
||||
print(f" speedup: {speedup:.2f}x")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Triton NVFP4 kernel requires CUDA.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"m, k",
|
||||
[
|
||||
(1, 16),
|
||||
(1, 4096),
|
||||
(2, 4096),
|
||||
(4, 4096),
|
||||
(8, 4096),
|
||||
(16, 4096),
|
||||
(24, 4096),
|
||||
(32, 4096),
|
||||
(1, 8192),
|
||||
(2, 8192),
|
||||
(4, 8192),
|
||||
(8, 8192),
|
||||
(16, 8192),
|
||||
(24, 8192),
|
||||
(32, 8192),
|
||||
(1, 32),
|
||||
(2, 48),
|
||||
(7, 64),
|
||||
(16, 128),
|
||||
(33, 160),
|
||||
(128, 256),
|
||||
(256, 512),
|
||||
(1024, 1024),
|
||||
(5120, 2048),
|
||||
(2048, 4096),
|
||||
(4096, 7168),
|
||||
(8192, 8192),
|
||||
(128, 16384),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("global_scale_value", [0.5, 1.0, 0.001])
|
||||
def test_triton_nvfp4_quant_dequant(
|
||||
monkeypatch, m: int, k: int, global_scale_value: float
|
||||
) -> None:
|
||||
"""Test the Triton quant-dequant kernel against the CPU reference."""
|
||||
block_size = 16
|
||||
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
|
||||
global_scale = torch.tensor(global_scale_value, dtype=torch.float32, device="cuda")
|
||||
|
||||
# Triton path
|
||||
triton_result = ref_nvfp4_quant_dequant(x, global_scale, block_size)
|
||||
|
||||
# CPU reference path
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setattr(
|
||||
nvfp4_emulation_utils.current_platform,
|
||||
"is_cuda_alike",
|
||||
lambda: False,
|
||||
)
|
||||
reference = ref_nvfp4_quant_dequant(x.cpu(), global_scale.cpu(), block_size)
|
||||
|
||||
torch.testing.assert_close(triton_result.cpu(), reference, atol=0, rtol=0)
|
||||
|
||||
# Benchmark (both paths on CUDA tensors for fair comparison)
|
||||
quantiles = [0.5, 0.001, 0.999]
|
||||
|
||||
def _triton_bench(
|
||||
input_tensor=x, input_global_scale=global_scale, input_block_size=block_size
|
||||
):
|
||||
return ref_nvfp4_quant_dequant(
|
||||
input_tensor, input_global_scale, input_block_size
|
||||
)
|
||||
|
||||
triton_ms, triton_min, triton_max = triton.testing.do_bench(
|
||||
_triton_bench, quantiles=quantiles
|
||||
)
|
||||
|
||||
def _reference_bench(
|
||||
input_tensor=x, input_global_scale=global_scale, input_block_size=block_size
|
||||
):
|
||||
with monkeypatch.context() as mp2:
|
||||
mp2.setattr(
|
||||
nvfp4_emulation_utils.current_platform,
|
||||
"is_cuda_alike",
|
||||
lambda: False,
|
||||
)
|
||||
ref_nvfp4_quant_dequant(input_tensor, input_global_scale, input_block_size)
|
||||
|
||||
ref_ms, ref_min, ref_max = triton.testing.do_bench(
|
||||
_reference_bench, quantiles=quantiles
|
||||
)
|
||||
|
||||
speedup = ref_ms / triton_ms if triton_ms > 0 else float("inf")
|
||||
print(f" quant_dequant [{m}x{k}] gs={global_scale_value}:")
|
||||
print(
|
||||
f" triton: median={triton_ms:.3f}ms, "
|
||||
f"min={triton_min:.3f}ms, max={triton_max:.3f}ms"
|
||||
)
|
||||
print(
|
||||
f" reference: median={ref_ms:.3f}ms, "
|
||||
f"min={ref_min:.3f}ms, max={ref_max:.3f}ms"
|
||||
)
|
||||
print(f" speedup: {speedup:.2f}x")
|
||||
@@ -31,6 +31,9 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_nvfp4_moe_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
kE2M1ToFloat_handle,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
)
|
||||
@@ -376,6 +379,10 @@ def convert_to_nvfp4_moe_kernel_format(
|
||||
is_act_and_mul=is_act_and_mul,
|
||||
)
|
||||
elif nvfp4_backend == NvFp4MoeBackend.EMULATION:
|
||||
# Move the E2M1 lookup table to the device now, because
|
||||
# `.to(device)` is not allowed during CUDA graph capture.
|
||||
kE2M1ToFloat_handle.val = kE2M1ToFloat_handle.val.to(layer.weight.device)
|
||||
|
||||
if a13_scale is None or a2_scale is None:
|
||||
raise ValueError(
|
||||
"Activation global scales should not be None, got"
|
||||
|
||||
@@ -298,7 +298,8 @@ def moe_kernel_quantize_input(
|
||||
A, A_scale, is_sf_swizzled_layout=is_fp4_scale_swizzled
|
||||
)
|
||||
else:
|
||||
return ref_nvfp4_quant_dequant(A, A_scale, block_size=16)
|
||||
A = ref_nvfp4_quant_dequant(A, A_scale, block_size=16)
|
||||
return A, None
|
||||
elif quant_dtype == "mxfp4":
|
||||
if not quantization_emulation:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -4,7 +4,9 @@ from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
__all__ = [
|
||||
"break_fp4_bytes",
|
||||
@@ -20,6 +22,309 @@ kE2M1ToFloat_handle = SimpleNamespace(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _e2m1_inline(magnitude):
|
||||
"""Inline E2M1 lookup using binary tree - 3 levels instead of 7 sequential.
|
||||
|
||||
Maps 3-bit magnitude to float: [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
|
||||
Uses bit decomposition for fewer comparisons.
|
||||
"""
|
||||
# Bit 2 (MSB): separates 0-3 from 4-7
|
||||
# Bit 1: separates within groups
|
||||
# Bit 0 (LSB): separates within pairs
|
||||
b2 = (magnitude >> 2) & 1 # 0 for mag 0-3, 1 for mag 4-7
|
||||
b1 = (magnitude >> 1) & 1 # middle bit
|
||||
b0 = magnitude & 1 # LSB
|
||||
|
||||
# For mag 0-3: [0.0, 0.5, 1.0, 1.5]
|
||||
low_group = tl.where(
|
||||
b1 == 1, tl.where(b0 == 1, 1.5, 1.0), tl.where(b0 == 1, 0.5, 0.0)
|
||||
)
|
||||
# For mag 4-7: [2.0, 3.0, 4.0, 6.0]
|
||||
high_group = tl.where(
|
||||
b1 == 1, tl.where(b0 == 1, 6.0, 4.0), tl.where(b0 == 1, 3.0, 2.0)
|
||||
)
|
||||
return tl.where(b2 == 1, high_group, low_group)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dequantize_nvfp4_kernel(
|
||||
fp4_ptr,
|
||||
scale_ptr,
|
||||
global_scale_ptr,
|
||||
output_ptr,
|
||||
rows_per_batch: tl.constexpr,
|
||||
num_blocks: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
has_batch_global_scale: tl.constexpr,
|
||||
TILE_BLOCKS: tl.constexpr,
|
||||
):
|
||||
"""Triton kernel for NVFP4 dequantization (swizzle=False).
|
||||
|
||||
Optimized with 2D tile processing + interleave for coalesced stores.
|
||||
"""
|
||||
BLOCK_PACKED: tl.constexpr = BLOCK_SIZE // 2
|
||||
|
||||
row_idx = tl.program_id(0)
|
||||
tile_idx = tl.program_id(1)
|
||||
|
||||
if has_batch_global_scale:
|
||||
batch_idx = row_idx // rows_per_batch
|
||||
global_scale = tl.load(global_scale_ptr + batch_idx).to(tl.float32)
|
||||
else:
|
||||
global_scale = tl.load(global_scale_ptr).to(tl.float32)
|
||||
|
||||
fp4_row_offset = row_idx * num_blocks * BLOCK_PACKED
|
||||
scale_row_offset = row_idx * num_blocks
|
||||
output_row_offset = row_idx * num_blocks * BLOCK_SIZE
|
||||
|
||||
start_block = tile_idx * TILE_BLOCKS
|
||||
|
||||
# Load scales for this tile: [TILE_BLOCKS]
|
||||
block_offsets = tl.arange(0, TILE_BLOCKS)
|
||||
block_mask = (start_block + block_offsets) < num_blocks
|
||||
|
||||
raw_scales = tl.load(
|
||||
scale_ptr + scale_row_offset + start_block + block_offsets,
|
||||
mask=block_mask,
|
||||
other=0,
|
||||
)
|
||||
scale_f32 = tl.cast(raw_scales, tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
scale_values = (scale_f32 * global_scale)[:, None]
|
||||
|
||||
# Load [TILE_BLOCKS, BLOCK_PACKED] packed bytes
|
||||
packed_offsets = tl.arange(0, BLOCK_PACKED)[None, :]
|
||||
byte_indices = (
|
||||
fp4_row_offset
|
||||
+ (start_block + block_offsets[:, None]) * BLOCK_PACKED
|
||||
+ packed_offsets
|
||||
)
|
||||
elem_mask = block_mask[:, None]
|
||||
raw_bytes = tl.load(fp4_ptr + byte_indices, mask=elem_mask, other=0)
|
||||
|
||||
low_nibble = raw_bytes & 0x0F
|
||||
high_nibble = (raw_bytes >> 4) & 0x0F
|
||||
|
||||
# Binary tree E2M1 decode
|
||||
low_mag = low_nibble & 0x07
|
||||
low_val = _e2m1_inline(low_mag)
|
||||
low_sign = (low_nibble >> 3) & 1
|
||||
low_result = tl.where(low_sign == 1, -low_val, low_val) * scale_values
|
||||
|
||||
high_mag = high_nibble & 0x07
|
||||
high_val = _e2m1_inline(high_mag)
|
||||
high_sign = (high_nibble >> 3) & 1
|
||||
high_result = tl.where(high_sign == 1, -high_val, high_val) * scale_values
|
||||
|
||||
# Interleave for coalesced contiguous store
|
||||
result = tl.interleave(low_result, high_result)
|
||||
|
||||
elem_offsets = tl.arange(0, BLOCK_SIZE)[None, :]
|
||||
out_indices = (
|
||||
output_row_offset
|
||||
+ (start_block + block_offsets[:, None]) * BLOCK_SIZE
|
||||
+ elem_offsets
|
||||
)
|
||||
tl.store(output_ptr + out_indices, result, mask=block_mask[:, None])
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _e2m1_lookup(magnitude):
|
||||
"""Lookup E2M1 float value from 3-bit magnitude."""
|
||||
result = tl.where(magnitude == 1, 0.5, 0.0)
|
||||
result = tl.where(magnitude == 2, 1.0, result)
|
||||
result = tl.where(magnitude == 3, 1.5, result)
|
||||
result = tl.where(magnitude == 4, 2.0, result)
|
||||
result = tl.where(magnitude == 5, 3.0, result)
|
||||
result = tl.where(magnitude == 6, 4.0, result)
|
||||
result = tl.where(magnitude == 7, 6.0, result)
|
||||
return result
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _round_to_fp4(x):
|
||||
"""Round float values to the nearest E2M1 representable value.
|
||||
|
||||
Matches the thresholds in the Python ``cast_to_fp4`` exactly.
|
||||
"""
|
||||
sign = tl.where(x < 0.0, -1.0, 1.0)
|
||||
abs_x = tl.abs(x)
|
||||
result = tl.where(abs_x > 5.0, 6.0, 0.0)
|
||||
result = tl.where((abs_x >= 3.5) & (abs_x <= 5.0), 4.0, result)
|
||||
result = tl.where((abs_x > 2.5) & (abs_x < 3.5), 3.0, result)
|
||||
result = tl.where((abs_x >= 1.75) & (abs_x <= 2.5), 2.0, result)
|
||||
result = tl.where((abs_x > 1.25) & (abs_x < 1.75), 1.5, result)
|
||||
result = tl.where((abs_x >= 0.75) & (abs_x <= 1.25), 1.0, result)
|
||||
result = tl.where((abs_x > 0.25) & (abs_x < 0.75), 0.5, result)
|
||||
return result * sign
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _nvfp4_quant_dequant_kernel(
|
||||
input_ptr,
|
||||
output_ptr,
|
||||
global_scale_ptr,
|
||||
k: tl.constexpr,
|
||||
num_blocks: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
FP4_MAX_RECIPROCAL: tl.constexpr,
|
||||
TILE_BLOCKS: tl.constexpr,
|
||||
):
|
||||
"""Fused NVFP4 quantize-dequantize kernel.
|
||||
|
||||
Uses a 2D grid (rows x tiles) to parallelize across both rows
|
||||
and quantization groups within a row. Each program handles
|
||||
TILE_BLOCKS groups at once using vectorized 2D operations.
|
||||
"""
|
||||
row_idx = tl.program_id(0)
|
||||
tile_idx = tl.program_id(1)
|
||||
global_scale = tl.load(global_scale_ptr).to(tl.float32)
|
||||
row_offset = row_idx * k
|
||||
|
||||
start_block = tile_idx * TILE_BLOCKS
|
||||
block_offsets = tl.arange(0, TILE_BLOCKS)
|
||||
block_mask = (start_block + block_offsets) < num_blocks
|
||||
|
||||
# Load [TILE_BLOCKS, BLOCK_SIZE] elements
|
||||
indices = (
|
||||
row_offset
|
||||
+ (start_block + block_offsets[:, None]) * BLOCK_SIZE
|
||||
+ tl.arange(0, BLOCK_SIZE)[None, :]
|
||||
)
|
||||
mask_2d = block_mask[:, None]
|
||||
x = tl.load(input_ptr + indices, mask=mask_2d, other=0.0).to(tl.float32)
|
||||
|
||||
# Per-group scale: [TILE_BLOCKS]
|
||||
vec_max = tl.max(tl.abs(x), axis=1)
|
||||
scale = global_scale * (vec_max * FP4_MAX_RECIPROCAL)
|
||||
scale = tl.clamp(scale, -448.0, 448.0)
|
||||
scale = scale.to(tl.float8e4nv).to(tl.float32)
|
||||
|
||||
# Safe reciprocal, broadcast to [TILE_BLOCKS, 1]
|
||||
output_scale = tl.where(scale == 0.0, 0.0, global_scale / scale)[:, None]
|
||||
|
||||
# Quantize: scale, clamp, round to FP4
|
||||
scaled_x = tl.clamp(x * output_scale, -6.0, 6.0)
|
||||
fp4_val = _round_to_fp4(scaled_x)
|
||||
|
||||
# Dequantize: fp4_val * (scale / global_scale)
|
||||
dequant_scale = (scale / global_scale)[:, None]
|
||||
result = fp4_val * dequant_scale
|
||||
|
||||
tl.store(output_ptr + indices, result, mask=mask_2d)
|
||||
|
||||
|
||||
def _triton_nvfp4_quant_dequant(
|
||||
x: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Triton-accelerated NVFP4 quantize-dequantize."""
|
||||
x_m, x_k = x.shape
|
||||
|
||||
if not torch.compiler.is_compiling():
|
||||
assert x_k % block_size == 0, (
|
||||
f"Weight shape K={x_k} is not divisible by block_size={block_size}"
|
||||
)
|
||||
|
||||
output_dtype = x.dtype
|
||||
num_blocks = x_k // block_size
|
||||
|
||||
output = torch.empty(x_m, x_k, dtype=output_dtype, device=x.device)
|
||||
|
||||
tile_blocks = min(64, triton.next_power_of_2(num_blocks))
|
||||
num_tiles = (num_blocks + tile_blocks - 1) // tile_blocks
|
||||
grid = (x_m, num_tiles)
|
||||
_nvfp4_quant_dequant_kernel[grid](
|
||||
x,
|
||||
output,
|
||||
global_scale,
|
||||
x_k,
|
||||
num_blocks,
|
||||
block_size,
|
||||
FLOAT4_E2M1_MAX_RECIPROCAL,
|
||||
tile_blocks,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def _triton_dequantize_nvfp4(
|
||||
tensor_fp4: torch.Tensor,
|
||||
tensor_sf: torch.Tensor,
|
||||
global_scale: torch.Tensor,
|
||||
dtype: torch.dtype,
|
||||
block_size: int = 16,
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize NVFP4 using Triton (swizzle=False only).
|
||||
|
||||
Supports both 2D and 3D inputs:
|
||||
- 2D: [m, packed_k] -> [m, k]
|
||||
- 3D: [dim0, m, packed_k] -> [dim0, m, k]
|
||||
"""
|
||||
assert tensor_fp4.dtype == torch.uint8
|
||||
|
||||
is_3d = tensor_fp4.ndim == 3
|
||||
if is_3d:
|
||||
dim0, m_per_batch, packed_k = tensor_fp4.shape
|
||||
tensor_fp4_2d = tensor_fp4.reshape(-1, packed_k)
|
||||
tensor_sf_2d = tensor_sf.reshape(-1, tensor_sf.shape[-1])
|
||||
total_rows_flat = dim0 * m_per_batch
|
||||
else:
|
||||
m_per_batch, packed_k = tensor_fp4.shape
|
||||
tensor_fp4_2d = tensor_fp4
|
||||
tensor_sf_2d = tensor_sf
|
||||
total_rows_flat = m_per_batch
|
||||
|
||||
k = packed_k * 2
|
||||
num_blocks = k // block_size
|
||||
|
||||
output = torch.empty(total_rows_flat, k, dtype=dtype, device=tensor_fp4.device)
|
||||
|
||||
# View as uint8 so Triton can load raw bytes and bitcast to float8_e4m3fn
|
||||
scale_raw = tensor_sf_2d.contiguous().view(torch.uint8)
|
||||
|
||||
# Shape-adaptive tile sizing: for large row counts (3D), process
|
||||
# entire row in one tile. For small row counts (2D), use smaller
|
||||
# tiles to increase parallelism across CUs.
|
||||
np2 = triton.next_power_of_2(num_blocks)
|
||||
if total_rows_flat >= 4096:
|
||||
# Many rows: maximize work per CTA, one tile per row
|
||||
tile_blocks = np2
|
||||
nw = 1
|
||||
ns = 2
|
||||
elif total_rows_flat >= 2048:
|
||||
# Medium-many rows: full row, 2 warps
|
||||
tile_blocks = np2
|
||||
nw = 2
|
||||
ns = 2
|
||||
else:
|
||||
# Few rows: use moderate tiles for CU utilization
|
||||
tile_blocks = min(64, np2)
|
||||
nw = 4
|
||||
ns = 2
|
||||
num_tiles = (num_blocks + tile_blocks - 1) // tile_blocks
|
||||
grid = (total_rows_flat, num_tiles)
|
||||
_dequantize_nvfp4_kernel[grid](
|
||||
tensor_fp4_2d,
|
||||
scale_raw,
|
||||
global_scale,
|
||||
output,
|
||||
m_per_batch,
|
||||
num_blocks,
|
||||
block_size,
|
||||
is_3d,
|
||||
tile_blocks,
|
||||
num_warps=nw,
|
||||
num_stages=ns,
|
||||
)
|
||||
|
||||
if is_3d:
|
||||
output = output.reshape(dim0, m_per_batch, k)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def break_fp4_bytes(a, dtype):
|
||||
assert a.dtype == torch.uint8
|
||||
m, n = a.shape
|
||||
@@ -67,6 +372,11 @@ def dequantize_to_dtype(
|
||||
# Two fp4 values are packed into one uint8.
|
||||
assert tensor_fp4.dtype == torch.uint8
|
||||
|
||||
if not swizzle and current_platform.is_cuda_alike():
|
||||
return _triton_dequantize_nvfp4(
|
||||
tensor_fp4, tensor_sf, global_scale, dtype, block_size
|
||||
)
|
||||
|
||||
# We handle 3D tensors reshaping them to 2D.
|
||||
is_3d = tensor_fp4.ndim == 3
|
||||
|
||||
@@ -145,12 +455,15 @@ def ref_nvfp4_quant(x, global_scale, block_size):
|
||||
|
||||
def ref_nvfp4_quant_dequant(
|
||||
x: torch.Tensor, global_scale: torch.Tensor, block_size: int
|
||||
) -> tuple[torch.Tensor, None]:
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
NVFP4 quantize-dequantize operation.
|
||||
|
||||
`global_scale` is expected to have a single element.
|
||||
"""
|
||||
if current_platform.is_cuda_alike():
|
||||
return _triton_nvfp4_quant_dequant(x, global_scale, block_size)
|
||||
|
||||
x_m, x_k = x.shape
|
||||
output_dtype = x.dtype
|
||||
|
||||
@@ -162,7 +475,7 @@ def ref_nvfp4_quant_dequant(
|
||||
x_blockscale = x_blockscale.unsqueeze(-1) / global_scale
|
||||
x_dq = (x_fp4 * x_blockscale).reshape(x_m, x_k).to(output_dtype)
|
||||
|
||||
return x_dq, None
|
||||
return x_dq
|
||||
|
||||
|
||||
def run_nvfp4_emulations(
|
||||
@@ -176,7 +489,7 @@ def run_nvfp4_emulations(
|
||||
output_dtype = x.dtype
|
||||
group_size = 16
|
||||
|
||||
x_dq, _ = ref_nvfp4_quant_dequant(x, input_global_scale, block_size=group_size)
|
||||
x_dq = ref_nvfp4_quant_dequant(x, input_global_scale, block_size=group_size)
|
||||
|
||||
# dequantize weight
|
||||
w_fp4 = weight.data.view(torch.uint8)
|
||||
|
||||
Reference in New Issue
Block a user