[NVFP4] Support NVFP4 dense models from modelopt and compressed-tensors on AMD Instinct MI300, MI355X and Hopper through emulation (#35733)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: fxmarty-amd <felmarty@amd.com>
Co-authored-by: Kyle Sayers <kylesayrs@gmail.com>
This commit is contained in:
fxmarty-amd
2026-04-06 16:18:27 -06:00
committed by GitHub
co-authored by Kyle Sayers
parent 9c81f35b1a
commit 00d7b497b3
10 changed files with 191 additions and 58 deletions
@@ -71,6 +71,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
NvFp4LinearBackend,
apply_nvfp4_linear,
convert_to_nvfp4_linear_kernel_format,
select_nvfp4_linear_backend,
@@ -1074,6 +1075,10 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
self.marlin_input_dtype = None
self.backend = select_nvfp4_linear_backend()
self.swizzle = None
if self.backend == NvFp4LinearBackend.EMULATION:
self.swizzle = False
def create_weights(
self,
layer: torch.nn.Module,
@@ -1149,10 +1154,23 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if (
torch.unique(layer.input_scale).numel() != 1
or torch.unique(layer.weight_scale_2).numel() != 1
):
logger.warning_once(
"In NVFP4 linear, the global scale for input or weight are different"
" for parallel layers (e.g. q_proj, k_proj, v_proj). This "
" will likely results in reduce accuracy. Please verify the model"
" accuracy. Consider using a checkpoint with a shared global NVFP4"
" scale for parallel layers."
)
# Rename ModelOpt checkpoint names to standardized names
input_global_scale = layer.input_scale.max().to(torch.float32)
layer.input_global_scale = Parameter(input_global_scale, requires_grad=False)
del layer.input_scale
weight_global_scale = layer.weight_scale_2.max().to(torch.float32)
layer.weight_global_scale = Parameter(weight_global_scale, requires_grad=False)
del layer.weight_scale_2
@@ -1179,6 +1197,7 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
layer=layer,
x=x,
bias=bias,
swizzle=self.swizzle,
)