[NVFP4] Support NVFP4 dense models from modelopt and compressed-tensors on AMD Instinct MI300, MI355X and Hopper through emulation (#35733)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: fxmarty-amd <felmarty@amd.com> Co-authored-by: Kyle Sayers <kylesayrs@gmail.com>
This commit is contained in:
co-authored by
Kyle Sayers
parent
9c81f35b1a
commit
00d7b497b3
@@ -71,6 +71,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
NvFp4LinearBackend,
|
||||
apply_nvfp4_linear,
|
||||
convert_to_nvfp4_linear_kernel_format,
|
||||
select_nvfp4_linear_backend,
|
||||
@@ -1074,6 +1075,10 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
|
||||
self.marlin_input_dtype = None
|
||||
self.backend = select_nvfp4_linear_backend()
|
||||
|
||||
self.swizzle = None
|
||||
if self.backend == NvFp4LinearBackend.EMULATION:
|
||||
self.swizzle = False
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -1149,10 +1154,23 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if (
|
||||
torch.unique(layer.input_scale).numel() != 1
|
||||
or torch.unique(layer.weight_scale_2).numel() != 1
|
||||
):
|
||||
logger.warning_once(
|
||||
"In NVFP4 linear, the global scale for input or weight are different"
|
||||
" for parallel layers (e.g. q_proj, k_proj, v_proj). This "
|
||||
" will likely results in reduce accuracy. Please verify the model"
|
||||
" accuracy. Consider using a checkpoint with a shared global NVFP4"
|
||||
" scale for parallel layers."
|
||||
)
|
||||
|
||||
# Rename ModelOpt checkpoint names to standardized names
|
||||
input_global_scale = layer.input_scale.max().to(torch.float32)
|
||||
layer.input_global_scale = Parameter(input_global_scale, requires_grad=False)
|
||||
del layer.input_scale
|
||||
|
||||
weight_global_scale = layer.weight_scale_2.max().to(torch.float32)
|
||||
layer.weight_global_scale = Parameter(weight_global_scale, requires_grad=False)
|
||||
del layer.weight_scale_2
|
||||
@@ -1179,6 +1197,7 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
|
||||
layer=layer,
|
||||
x=x,
|
||||
bias=bias,
|
||||
swizzle=self.swizzle,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user