[Bugfix/Core] Flashinfer k_scale and v_scale (#9861)

This commit is contained in:
Pavani Majety
2024-11-01 12:15:05 -07:00
committed by GitHub
parent aff1fd8188
commit 598b6d7b07
3 changed files with 25 additions and 12 deletions
@@ -141,8 +141,11 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
layer.register_parameter("input_scale", scale)
def process_weights_after_loading(self, layer: Module) -> None:
max_w_scale, weight = requantize_with_max_scale(
layer.weight, layer.weight_scale, layer.logical_widths)
weight = layer.weight
max_w_scale = layer.weight_scale.max()
if not (layer.weight_scale == layer.weight_scale[0]).all():
max_w_scale, weight = requantize_with_max_scale(
layer.weight, layer.weight_scale, layer.logical_widths)
layer.weight = Parameter(weight.t(), requires_grad=False)
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
layer.input_scale = Parameter(layer.input_scale.max(),