diff --git a/tests/kernels/quantization/test_nvfp4_emulation.py b/tests/kernels/quantization/test_nvfp4_emulation.py new file mode 100644 index 0000000000..71072d9e9f --- /dev/null +++ b/tests/kernels/quantization/test_nvfp4_emulation.py @@ -0,0 +1,308 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import huggingface_hub +import pytest +import torch +from safetensors import safe_open + +from vllm.model_executor.layers.quantization.utils import ( + nvfp4_emulation_utils, +) +from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( + dequantize_to_dtype, + ref_nvfp4_quant_dequant, +) +from vllm.platforms import current_platform +from vllm.triton_utils import triton + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), + reason="Triton NVFP4 kernel requires CUDA.", +) +def test_triton_dequantize_nvfp4(monkeypatch) -> None: + """Test the Triton dequantization kernel against the CPU reference + using real NVFP4 weights from a checkpoint. + + Tests both 2D (attention projection) and 3D (stacked MoE experts). + """ + checkpoint_path = huggingface_hub.snapshot_download( + "nvidia/Qwen3-30B-A3B-NVFP4", + allow_patterns=["model-00001-of-00004.safetensors"], + ) + shard_path = f"{checkpoint_path}/model-00001-of-00004.safetensors" + block_size = 16 + + with safe_open(shard_path, framework="pt", device="cpu") as f: + all_keys = list(f.keys()) + + # 2D case: attention projection + tensor_fp4_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight") + tensor_sf_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight_scale") + global_scale_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight_scale_2") + + # 3D case: stack ALL experts for layer 9 up_proj + expert_prefix = "model.layers.9.mlp.experts." + expert_indices = sorted( + int(key.split(".")[5]) + for key in all_keys + if key.startswith(expert_prefix) and key.endswith(".up_proj.weight") + ) + assert len(expert_indices) > 0 + + all_fp4 = [] + all_sf = [] + all_global_scale = [] + for index in expert_indices: + name = f"{expert_prefix}{index}.up_proj" + all_fp4.append(f.get_tensor(f"{name}.weight")) + all_sf.append(f.get_tensor(f"{name}.weight_scale")) + all_global_scale.append(f.get_tensor(f"{name}.weight_scale_2")) + + tensor_fp4_3d = torch.stack(all_fp4) + tensor_sf_3d = torch.stack(all_sf) + global_scale_3d = torch.stack(all_global_scale) + + test_cases = [ + ("2D base", tensor_fp4_2d, tensor_sf_2d, global_scale_2d), + ( + "2D 2x rows", + tensor_fp4_2d.repeat(2, 1), + tensor_sf_2d.repeat(2, 1), + global_scale_2d, + ), + ( + "2D 4x rows", + tensor_fp4_2d.repeat(4, 1), + tensor_sf_2d.repeat(4, 1), + global_scale_2d, + ), + ( + "2D 2x cols", + tensor_fp4_2d.repeat(1, 2), + tensor_sf_2d.repeat(1, 2), + global_scale_2d, + ), + ("3D base", tensor_fp4_3d, tensor_sf_3d, global_scale_3d), + ( + "3D 2x experts", + tensor_fp4_3d.repeat(2, 1, 1), + tensor_sf_3d.repeat(2, 1, 1), + global_scale_3d.repeat(2), + ), + ( + "3D 2x rows", + tensor_fp4_3d.repeat(1, 2, 1), + tensor_sf_3d.repeat(1, 2, 1), + global_scale_3d, + ), + ( + "3D 2x cols", + tensor_fp4_3d.repeat(1, 1, 2), + tensor_sf_3d.repeat(1, 1, 2), + global_scale_3d, + ), + ] + + quantiles = [0.5, 0.001, 0.999] + + # Move the E2M1 lookup table to CUDA ahead of time, as would normally + # happen during model loading (process_weights_after_loading). Both the + # Triton and PyTorch reference paths run on CUDA. + nvfp4_emulation_utils.kE2M1ToFloat_handle.val = ( + nvfp4_emulation_utils.kE2M1ToFloat_handle.val.cuda() + ) + + for label, tensor_fp4, tensor_sf, global_scale in test_cases: + fp4_cuda = tensor_fp4.cuda() + sf_cuda = tensor_sf.cuda() + gs_cuda = global_scale.cuda() + + # Triton path + triton_result = dequantize_to_dtype( + fp4_cuda, + sf_cuda, + gs_cuda, + torch.bfloat16, + block_size, + swizzle=False, + ) + + # Reference path (PyTorch ops on CUDA, Triton dispatch disabled) + with monkeypatch.context() as m: + m.setattr( + nvfp4_emulation_utils.current_platform, + "is_cuda_alike", + lambda: False, + ) + reference = dequantize_to_dtype( + fp4_cuda, + sf_cuda, + gs_cuda, + torch.bfloat16, + block_size, + swizzle=False, + ) + + torch.testing.assert_close(triton_result, reference, atol=0, rtol=0) + + # Benchmark + shape = list(tensor_fp4.shape) + + def _triton_bench( + fp4_cuda=fp4_cuda, + scale_cuda=sf_cuda, + global_scale_cuda=gs_cuda, + block_size=block_size, + ): + return dequantize_to_dtype( + fp4_cuda, + scale_cuda, + global_scale_cuda, + torch.bfloat16, + block_size, + swizzle=False, + ) + + triton_ms, triton_min, triton_max = triton.testing.do_bench( + _triton_bench, quantiles=quantiles + ) + + def _reference_bench( + fp4_cuda=fp4_cuda, + scale_cuda=sf_cuda, + global_scale_cuda=gs_cuda, + block_size=block_size, + ): + with monkeypatch.context() as m2: + m2.setattr( + nvfp4_emulation_utils.current_platform, + "is_cuda_alike", + lambda: False, + ) + dequantize_to_dtype( + fp4_cuda, + scale_cuda, + global_scale_cuda, + torch.bfloat16, + block_size, + swizzle=False, + ) + + ref_ms, ref_min, ref_max = triton.testing.do_bench( + _reference_bench, quantiles=quantiles + ) + + speedup = ref_ms / triton_ms if triton_ms > 0 else float("inf") + print(f" dequantize {label} {shape}:") + print( + f" triton: median={triton_ms:.3f}ms, " + f"min={triton_min:.3f}ms, max={triton_max:.3f}ms" + ) + print( + f" reference: median={ref_ms:.3f}ms, " + f"min={ref_min:.3f}ms, max={ref_max:.3f}ms" + ) + print(f" speedup: {speedup:.2f}x") + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), + reason="Triton NVFP4 kernel requires CUDA.", +) +@pytest.mark.parametrize( + "m, k", + [ + (1, 16), + (1, 4096), + (2, 4096), + (4, 4096), + (8, 4096), + (16, 4096), + (24, 4096), + (32, 4096), + (1, 8192), + (2, 8192), + (4, 8192), + (8, 8192), + (16, 8192), + (24, 8192), + (32, 8192), + (1, 32), + (2, 48), + (7, 64), + (16, 128), + (33, 160), + (128, 256), + (256, 512), + (1024, 1024), + (5120, 2048), + (2048, 4096), + (4096, 7168), + (8192, 8192), + (128, 16384), + ], +) +@pytest.mark.parametrize("global_scale_value", [0.5, 1.0, 0.001]) +def test_triton_nvfp4_quant_dequant( + monkeypatch, m: int, k: int, global_scale_value: float +) -> None: + """Test the Triton quant-dequant kernel against the CPU reference.""" + block_size = 16 + x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda") + global_scale = torch.tensor(global_scale_value, dtype=torch.float32, device="cuda") + + # Triton path + triton_result = ref_nvfp4_quant_dequant(x, global_scale, block_size) + + # CPU reference path + with monkeypatch.context() as mp: + mp.setattr( + nvfp4_emulation_utils.current_platform, + "is_cuda_alike", + lambda: False, + ) + reference = ref_nvfp4_quant_dequant(x.cpu(), global_scale.cpu(), block_size) + + torch.testing.assert_close(triton_result.cpu(), reference, atol=0, rtol=0) + + # Benchmark (both paths on CUDA tensors for fair comparison) + quantiles = [0.5, 0.001, 0.999] + + def _triton_bench( + input_tensor=x, input_global_scale=global_scale, input_block_size=block_size + ): + return ref_nvfp4_quant_dequant( + input_tensor, input_global_scale, input_block_size + ) + + triton_ms, triton_min, triton_max = triton.testing.do_bench( + _triton_bench, quantiles=quantiles + ) + + def _reference_bench( + input_tensor=x, input_global_scale=global_scale, input_block_size=block_size + ): + with monkeypatch.context() as mp2: + mp2.setattr( + nvfp4_emulation_utils.current_platform, + "is_cuda_alike", + lambda: False, + ) + ref_nvfp4_quant_dequant(input_tensor, input_global_scale, input_block_size) + + ref_ms, ref_min, ref_max = triton.testing.do_bench( + _reference_bench, quantiles=quantiles + ) + + speedup = ref_ms / triton_ms if triton_ms > 0 else float("inf") + print(f" quant_dequant [{m}x{k}] gs={global_scale_value}:") + print( + f" triton: median={triton_ms:.3f}ms, " + f"min={triton_min:.3f}ms, max={triton_max:.3f}ms" + ) + print( + f" reference: median={ref_ms:.3f}ms, " + f"min={ref_min:.3f}ms, max={ref_max:.3f}ms" + ) + print(f" speedup: {speedup:.2f}x") diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index db6d56e3c3..01ac5cfa9d 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -31,6 +31,9 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( prepare_nvfp4_moe_layer_for_marlin, ) +from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( + kE2M1ToFloat_handle, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, ) @@ -376,6 +379,10 @@ def convert_to_nvfp4_moe_kernel_format( is_act_and_mul=is_act_and_mul, ) elif nvfp4_backend == NvFp4MoeBackend.EMULATION: + # Move the E2M1 lookup table to the device now, because + # `.to(device)` is not allowed during CUDA graph capture. + kE2M1ToFloat_handle.val = kE2M1ToFloat_handle.val.to(layer.weight.device) + if a13_scale is None or a2_scale is None: raise ValueError( "Activation global scales should not be None, got" diff --git a/vllm/model_executor/layers/fused_moe/utils.py b/vllm/model_executor/layers/fused_moe/utils.py index ffab3ca0bf..ee0a7657a4 100644 --- a/vllm/model_executor/layers/fused_moe/utils.py +++ b/vllm/model_executor/layers/fused_moe/utils.py @@ -298,7 +298,8 @@ def moe_kernel_quantize_input( A, A_scale, is_sf_swizzled_layout=is_fp4_scale_swizzled ) else: - return ref_nvfp4_quant_dequant(A, A_scale, block_size=16) + A = ref_nvfp4_quant_dequant(A, A_scale, block_size=16) + return A, None elif quant_dtype == "mxfp4": if not quantization_emulation: raise NotImplementedError( diff --git a/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py b/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py index af5c6f2a7a..39c78a9062 100644 --- a/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py +++ b/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py @@ -4,7 +4,9 @@ from types import SimpleNamespace import torch +from vllm.platforms import current_platform from vllm.scalar_type import scalar_types +from vllm.triton_utils import tl, triton __all__ = [ "break_fp4_bytes", @@ -20,6 +22,309 @@ kE2M1ToFloat_handle = SimpleNamespace( ) +@triton.jit +def _e2m1_inline(magnitude): + """Inline E2M1 lookup using binary tree - 3 levels instead of 7 sequential. + + Maps 3-bit magnitude to float: [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] + Uses bit decomposition for fewer comparisons. + """ + # Bit 2 (MSB): separates 0-3 from 4-7 + # Bit 1: separates within groups + # Bit 0 (LSB): separates within pairs + b2 = (magnitude >> 2) & 1 # 0 for mag 0-3, 1 for mag 4-7 + b1 = (magnitude >> 1) & 1 # middle bit + b0 = magnitude & 1 # LSB + + # For mag 0-3: [0.0, 0.5, 1.0, 1.5] + low_group = tl.where( + b1 == 1, tl.where(b0 == 1, 1.5, 1.0), tl.where(b0 == 1, 0.5, 0.0) + ) + # For mag 4-7: [2.0, 3.0, 4.0, 6.0] + high_group = tl.where( + b1 == 1, tl.where(b0 == 1, 6.0, 4.0), tl.where(b0 == 1, 3.0, 2.0) + ) + return tl.where(b2 == 1, high_group, low_group) + + +@triton.jit +def _dequantize_nvfp4_kernel( + fp4_ptr, + scale_ptr, + global_scale_ptr, + output_ptr, + rows_per_batch: tl.constexpr, + num_blocks: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + has_batch_global_scale: tl.constexpr, + TILE_BLOCKS: tl.constexpr, +): + """Triton kernel for NVFP4 dequantization (swizzle=False). + + Optimized with 2D tile processing + interleave for coalesced stores. + """ + BLOCK_PACKED: tl.constexpr = BLOCK_SIZE // 2 + + row_idx = tl.program_id(0) + tile_idx = tl.program_id(1) + + if has_batch_global_scale: + batch_idx = row_idx // rows_per_batch + global_scale = tl.load(global_scale_ptr + batch_idx).to(tl.float32) + else: + global_scale = tl.load(global_scale_ptr).to(tl.float32) + + fp4_row_offset = row_idx * num_blocks * BLOCK_PACKED + scale_row_offset = row_idx * num_blocks + output_row_offset = row_idx * num_blocks * BLOCK_SIZE + + start_block = tile_idx * TILE_BLOCKS + + # Load scales for this tile: [TILE_BLOCKS] + block_offsets = tl.arange(0, TILE_BLOCKS) + block_mask = (start_block + block_offsets) < num_blocks + + raw_scales = tl.load( + scale_ptr + scale_row_offset + start_block + block_offsets, + mask=block_mask, + other=0, + ) + scale_f32 = tl.cast(raw_scales, tl.float8e4nv, bitcast=True).to(tl.float32) + scale_values = (scale_f32 * global_scale)[:, None] + + # Load [TILE_BLOCKS, BLOCK_PACKED] packed bytes + packed_offsets = tl.arange(0, BLOCK_PACKED)[None, :] + byte_indices = ( + fp4_row_offset + + (start_block + block_offsets[:, None]) * BLOCK_PACKED + + packed_offsets + ) + elem_mask = block_mask[:, None] + raw_bytes = tl.load(fp4_ptr + byte_indices, mask=elem_mask, other=0) + + low_nibble = raw_bytes & 0x0F + high_nibble = (raw_bytes >> 4) & 0x0F + + # Binary tree E2M1 decode + low_mag = low_nibble & 0x07 + low_val = _e2m1_inline(low_mag) + low_sign = (low_nibble >> 3) & 1 + low_result = tl.where(low_sign == 1, -low_val, low_val) * scale_values + + high_mag = high_nibble & 0x07 + high_val = _e2m1_inline(high_mag) + high_sign = (high_nibble >> 3) & 1 + high_result = tl.where(high_sign == 1, -high_val, high_val) * scale_values + + # Interleave for coalesced contiguous store + result = tl.interleave(low_result, high_result) + + elem_offsets = tl.arange(0, BLOCK_SIZE)[None, :] + out_indices = ( + output_row_offset + + (start_block + block_offsets[:, None]) * BLOCK_SIZE + + elem_offsets + ) + tl.store(output_ptr + out_indices, result, mask=block_mask[:, None]) + + +@triton.jit +def _e2m1_lookup(magnitude): + """Lookup E2M1 float value from 3-bit magnitude.""" + result = tl.where(magnitude == 1, 0.5, 0.0) + result = tl.where(magnitude == 2, 1.0, result) + result = tl.where(magnitude == 3, 1.5, result) + result = tl.where(magnitude == 4, 2.0, result) + result = tl.where(magnitude == 5, 3.0, result) + result = tl.where(magnitude == 6, 4.0, result) + result = tl.where(magnitude == 7, 6.0, result) + return result + + +@triton.jit +def _round_to_fp4(x): + """Round float values to the nearest E2M1 representable value. + + Matches the thresholds in the Python ``cast_to_fp4`` exactly. + """ + sign = tl.where(x < 0.0, -1.0, 1.0) + abs_x = tl.abs(x) + result = tl.where(abs_x > 5.0, 6.0, 0.0) + result = tl.where((abs_x >= 3.5) & (abs_x <= 5.0), 4.0, result) + result = tl.where((abs_x > 2.5) & (abs_x < 3.5), 3.0, result) + result = tl.where((abs_x >= 1.75) & (abs_x <= 2.5), 2.0, result) + result = tl.where((abs_x > 1.25) & (abs_x < 1.75), 1.5, result) + result = tl.where((abs_x >= 0.75) & (abs_x <= 1.25), 1.0, result) + result = tl.where((abs_x > 0.25) & (abs_x < 0.75), 0.5, result) + return result * sign + + +@triton.jit +def _nvfp4_quant_dequant_kernel( + input_ptr, + output_ptr, + global_scale_ptr, + k: tl.constexpr, + num_blocks: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + FP4_MAX_RECIPROCAL: tl.constexpr, + TILE_BLOCKS: tl.constexpr, +): + """Fused NVFP4 quantize-dequantize kernel. + + Uses a 2D grid (rows x tiles) to parallelize across both rows + and quantization groups within a row. Each program handles + TILE_BLOCKS groups at once using vectorized 2D operations. + """ + row_idx = tl.program_id(0) + tile_idx = tl.program_id(1) + global_scale = tl.load(global_scale_ptr).to(tl.float32) + row_offset = row_idx * k + + start_block = tile_idx * TILE_BLOCKS + block_offsets = tl.arange(0, TILE_BLOCKS) + block_mask = (start_block + block_offsets) < num_blocks + + # Load [TILE_BLOCKS, BLOCK_SIZE] elements + indices = ( + row_offset + + (start_block + block_offsets[:, None]) * BLOCK_SIZE + + tl.arange(0, BLOCK_SIZE)[None, :] + ) + mask_2d = block_mask[:, None] + x = tl.load(input_ptr + indices, mask=mask_2d, other=0.0).to(tl.float32) + + # Per-group scale: [TILE_BLOCKS] + vec_max = tl.max(tl.abs(x), axis=1) + scale = global_scale * (vec_max * FP4_MAX_RECIPROCAL) + scale = tl.clamp(scale, -448.0, 448.0) + scale = scale.to(tl.float8e4nv).to(tl.float32) + + # Safe reciprocal, broadcast to [TILE_BLOCKS, 1] + output_scale = tl.where(scale == 0.0, 0.0, global_scale / scale)[:, None] + + # Quantize: scale, clamp, round to FP4 + scaled_x = tl.clamp(x * output_scale, -6.0, 6.0) + fp4_val = _round_to_fp4(scaled_x) + + # Dequantize: fp4_val * (scale / global_scale) + dequant_scale = (scale / global_scale)[:, None] + result = fp4_val * dequant_scale + + tl.store(output_ptr + indices, result, mask=mask_2d) + + +def _triton_nvfp4_quant_dequant( + x: torch.Tensor, + global_scale: torch.Tensor, + block_size: int, +) -> torch.Tensor: + """Triton-accelerated NVFP4 quantize-dequantize.""" + x_m, x_k = x.shape + + if not torch.compiler.is_compiling(): + assert x_k % block_size == 0, ( + f"Weight shape K={x_k} is not divisible by block_size={block_size}" + ) + + output_dtype = x.dtype + num_blocks = x_k // block_size + + output = torch.empty(x_m, x_k, dtype=output_dtype, device=x.device) + + tile_blocks = min(64, triton.next_power_of_2(num_blocks)) + num_tiles = (num_blocks + tile_blocks - 1) // tile_blocks + grid = (x_m, num_tiles) + _nvfp4_quant_dequant_kernel[grid]( + x, + output, + global_scale, + x_k, + num_blocks, + block_size, + FLOAT4_E2M1_MAX_RECIPROCAL, + tile_blocks, + ) + + return output + + +def _triton_dequantize_nvfp4( + tensor_fp4: torch.Tensor, + tensor_sf: torch.Tensor, + global_scale: torch.Tensor, + dtype: torch.dtype, + block_size: int = 16, +) -> torch.Tensor: + """Dequantize NVFP4 using Triton (swizzle=False only). + + Supports both 2D and 3D inputs: + - 2D: [m, packed_k] -> [m, k] + - 3D: [dim0, m, packed_k] -> [dim0, m, k] + """ + assert tensor_fp4.dtype == torch.uint8 + + is_3d = tensor_fp4.ndim == 3 + if is_3d: + dim0, m_per_batch, packed_k = tensor_fp4.shape + tensor_fp4_2d = tensor_fp4.reshape(-1, packed_k) + tensor_sf_2d = tensor_sf.reshape(-1, tensor_sf.shape[-1]) + total_rows_flat = dim0 * m_per_batch + else: + m_per_batch, packed_k = tensor_fp4.shape + tensor_fp4_2d = tensor_fp4 + tensor_sf_2d = tensor_sf + total_rows_flat = m_per_batch + + k = packed_k * 2 + num_blocks = k // block_size + + output = torch.empty(total_rows_flat, k, dtype=dtype, device=tensor_fp4.device) + + # View as uint8 so Triton can load raw bytes and bitcast to float8_e4m3fn + scale_raw = tensor_sf_2d.contiguous().view(torch.uint8) + + # Shape-adaptive tile sizing: for large row counts (3D), process + # entire row in one tile. For small row counts (2D), use smaller + # tiles to increase parallelism across CUs. + np2 = triton.next_power_of_2(num_blocks) + if total_rows_flat >= 4096: + # Many rows: maximize work per CTA, one tile per row + tile_blocks = np2 + nw = 1 + ns = 2 + elif total_rows_flat >= 2048: + # Medium-many rows: full row, 2 warps + tile_blocks = np2 + nw = 2 + ns = 2 + else: + # Few rows: use moderate tiles for CU utilization + tile_blocks = min(64, np2) + nw = 4 + ns = 2 + num_tiles = (num_blocks + tile_blocks - 1) // tile_blocks + grid = (total_rows_flat, num_tiles) + _dequantize_nvfp4_kernel[grid]( + tensor_fp4_2d, + scale_raw, + global_scale, + output, + m_per_batch, + num_blocks, + block_size, + is_3d, + tile_blocks, + num_warps=nw, + num_stages=ns, + ) + + if is_3d: + output = output.reshape(dim0, m_per_batch, k) + + return output + + def break_fp4_bytes(a, dtype): assert a.dtype == torch.uint8 m, n = a.shape @@ -67,6 +372,11 @@ def dequantize_to_dtype( # Two fp4 values are packed into one uint8. assert tensor_fp4.dtype == torch.uint8 + if not swizzle and current_platform.is_cuda_alike(): + return _triton_dequantize_nvfp4( + tensor_fp4, tensor_sf, global_scale, dtype, block_size + ) + # We handle 3D tensors reshaping them to 2D. is_3d = tensor_fp4.ndim == 3 @@ -145,12 +455,15 @@ def ref_nvfp4_quant(x, global_scale, block_size): def ref_nvfp4_quant_dequant( x: torch.Tensor, global_scale: torch.Tensor, block_size: int -) -> tuple[torch.Tensor, None]: +) -> torch.Tensor: """ NVFP4 quantize-dequantize operation. `global_scale` is expected to have a single element. """ + if current_platform.is_cuda_alike(): + return _triton_nvfp4_quant_dequant(x, global_scale, block_size) + x_m, x_k = x.shape output_dtype = x.dtype @@ -162,7 +475,7 @@ def ref_nvfp4_quant_dequant( x_blockscale = x_blockscale.unsqueeze(-1) / global_scale x_dq = (x_fp4 * x_blockscale).reshape(x_m, x_k).to(output_dtype) - return x_dq, None + return x_dq def run_nvfp4_emulations( @@ -176,7 +489,7 @@ def run_nvfp4_emulations( output_dtype = x.dtype group_size = 16 - x_dq, _ = ref_nvfp4_quant_dequant(x, input_global_scale, block_size=group_size) + x_dq = ref_nvfp4_quant_dequant(x, input_global_scale, block_size=group_size) # dequantize weight w_fp4 = weight.data.view(torch.uint8)