[ Kernel ] Enable fp8-marlin for fbgemm-fp8 models (#6606)

This commit is contained in:
Robert Shaw
2024-07-20 18:50:10 +00:00
committed by GitHub
parent 06d6c5fe9f
commit 9364f74eee
4 changed files with 44 additions and 3 deletions
@@ -76,8 +76,13 @@ def prepare_fp8_layer_for_marlin(layer: torch.nn.Module) -> None:
# WEIGHT SCALES
# Currently Marlin doesn't support per-tensor scales, so we
# expand it to channelwise
scales = layer.weight_scale.repeat(1, part_size_n).to(
layer.orig_dtype).to(device)
is_channelwise = layer.weight_scale.shape[0] == part_size_n
if is_channelwise:
scales = layer.weight_scale
else:
scales = layer.weight_scale.repeat(1, part_size_n)
scales = scales.to(layer.orig_dtype).to(device)
# Permute scales
marlin_scales = marlin_permute_scales(s=scales,
size_k=part_size_k,