[Bugfix][Quantization] Support CT block FP8 with Marlin (#52966)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
cb58bb9c1e
commit
541c6d64c1
@@ -57,14 +57,21 @@ class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
self.block_quant = self.config.weight_quant_key in {kFp8Static128BlockSym}
|
||||
self.size_k_first = not self.block_quant
|
||||
|
||||
@staticmethod
|
||||
def _block_scale_name(layer: torch.nn.Module) -> str:
|
||||
if getattr(layer, "weight_scale_inv", None) is not None:
|
||||
return "weight_scale_inv"
|
||||
return "weight_scale"
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if self.block_quant:
|
||||
weight, weight_scale_inv = process_fp8_weight_block_strategy(
|
||||
layer.weight, layer.weight_scale_inv
|
||||
scale_name = self._block_scale_name(layer)
|
||||
weight, weight_scale = process_fp8_weight_block_strategy(
|
||||
layer.weight, getattr(layer, scale_name)
|
||||
)
|
||||
# Update layer with new values
|
||||
replace_parameter(layer, "weight", weight.data)
|
||||
replace_parameter(layer, "weight_scale_inv", weight_scale_inv.data)
|
||||
replace_parameter(layer, scale_name, weight_scale.data)
|
||||
# Non-block: callers must pass weight in (K, N) layout.
|
||||
|
||||
layer.input_scale = None
|
||||
@@ -80,7 +87,7 @@ class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.block_quant:
|
||||
weight_scale = layer.weight_scale_inv
|
||||
weight_scale = getattr(layer, self._block_scale_name(layer))
|
||||
else:
|
||||
weight_scale = layer.weight_scale
|
||||
return apply_fp8_marlin_linear(
|
||||
|
||||
Reference in New Issue
Block a user