[Bugfix/Core] Flashinfer k_scale and v_scale (#9861)
This commit is contained in:
@@ -141,8 +141,11 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
|
||||
layer.register_parameter("input_scale", scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
max_w_scale, weight = requantize_with_max_scale(
|
||||
layer.weight, layer.weight_scale, layer.logical_widths)
|
||||
weight = layer.weight
|
||||
max_w_scale = layer.weight_scale.max()
|
||||
if not (layer.weight_scale == layer.weight_scale[0]).all():
|
||||
max_w_scale, weight = requantize_with_max_scale(
|
||||
layer.weight, layer.weight_scale, layer.logical_widths)
|
||||
layer.weight = Parameter(weight.t(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
|
||||
layer.input_scale = Parameter(layer.input_scale.max(),
|
||||
|
||||
Reference in New Issue
Block a user