[Bugfix] Remove NVFP4 scales assertions to fix load_format=dummy (#18861)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -22,7 +22,12 @@ def is_fp4_marlin_supported():
|
||||
|
||||
|
||||
def fp4_marlin_process_scales(marlin_scales):
|
||||
assert (marlin_scales >= 0).all()
|
||||
if not (marlin_scales >= 0).all():
|
||||
logger.warning_once(
|
||||
"NVFP4 Marlin assumes the scales to be >=0, but has encountered "
|
||||
"negative scales. Accuracy will likely be degraded. This is "
|
||||
"because it changes the scales from FP8-S1E4M3 to a special "
|
||||
"FP8-S0E5M3 format to speedup the dequantization.")
|
||||
|
||||
# convert to half first, we would convert to fp8 later
|
||||
marlin_scales = marlin_scales.to(torch.half)
|
||||
|
||||
Reference in New Issue
Block a user