[Bugfix] Fix DeepGEMM FP8 warmup coverage (#56452)
Signed-off-by: Ayushman Singh <40520701+ayush1399@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
29332cf936
commit
7bcca16729
@@ -38,6 +38,22 @@ class FP8BlockParams(FP8Params):
|
||||
input_scale_ub=getattr(layer, cls.INPUT_SCALE_UB, None),
|
||||
)
|
||||
|
||||
@property
|
||||
def block_scale_attr(self) -> str:
|
||||
"""Fp8LinearMethod registers the block scale as ``weight_scale_inv``,
|
||||
compressed-tensors as ``weight_scale``."""
|
||||
return (
|
||||
self.WEIGHT_SCALE
|
||||
if self.weight_scale_inv is None
|
||||
else self.WEIGHT_SCALE_INV
|
||||
)
|
||||
|
||||
@property
|
||||
def block_scale(self) -> torch.Tensor:
|
||||
scale = getattr(self, self.block_scale_attr)
|
||||
assert scale is not None
|
||||
return scale
|
||||
|
||||
|
||||
class Fp8BlockScaledMMLinearKernel(
|
||||
MMLinearKernel[FP8ScaledMMLinearLayerConfig, FP8BlockParams], ABC
|
||||
@@ -74,25 +90,13 @@ class Fp8BlockScaledMMLinearKernel(
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
params = self._get_layer_params(layer)
|
||||
# Fp8LinearMethod registered weight scale
|
||||
# buffer as weight_scale_inv unlike compressed tensors.
|
||||
weight_scale = (
|
||||
params.weight_scale
|
||||
if params.weight_scale_inv is None
|
||||
else params.weight_scale_inv
|
||||
)
|
||||
scale_attr_name = (
|
||||
params.WEIGHT_SCALE
|
||||
if params.weight_scale_inv is None
|
||||
else params.WEIGHT_SCALE_INV
|
||||
)
|
||||
new_weight, new_weight_scale = process_fp8_weight_block_strategy(
|
||||
params.weight,
|
||||
weight_scale,
|
||||
params.block_scale,
|
||||
)
|
||||
|
||||
replace_parameter(layer, params.WEIGHT, new_weight.data)
|
||||
replace_parameter(layer, scale_attr_name, new_weight_scale.data)
|
||||
replace_parameter(layer, params.block_scale_attr, new_weight_scale.data)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -104,11 +108,7 @@ class Fp8BlockScaledMMLinearKernel(
|
||||
out_dtype = self.config.out_dtype
|
||||
params = self._get_layer_params(layer)
|
||||
weight = params.weight
|
||||
weight_scale = (
|
||||
params.weight_scale
|
||||
if params.weight_scale_inv is None
|
||||
else params.weight_scale_inv
|
||||
)
|
||||
weight_scale = params.block_scale
|
||||
input_scale = params.input_scale
|
||||
scale_up = params.input_scale_ub
|
||||
|
||||
|
||||
@@ -86,25 +86,28 @@ class DeepGemmFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
params = self._get_layer_params(layer)
|
||||
assert layer.weight_block_size is not None
|
||||
|
||||
is_bmm = getattr(layer, "is_bmm", False)
|
||||
|
||||
if self.is_deep_gemm_supported:
|
||||
weight_scale_invs = params.weight_scale_inv
|
||||
scale_attr = (
|
||||
params.WEIGHT_SCALE_INV
|
||||
if weight_scale_invs is not None
|
||||
else params.WEIGHT_SCALE
|
||||
)
|
||||
dg_weight, dg_weight_scale = deepgemm_post_process_fp8_weight_block(
|
||||
wq=params.weight,
|
||||
ws=weight_scale_invs
|
||||
if weight_scale_invs is not None
|
||||
else params.weight_scale,
|
||||
ws=params.block_scale,
|
||||
quant_block_shape=tuple(layer.weight_block_size),
|
||||
use_e8m0=self.use_deep_gemm_e8m0,
|
||||
is_bmm=getattr(layer, "is_bmm", False),
|
||||
is_bmm=is_bmm,
|
||||
bmm_batch_size=getattr(layer, "bmm_batch_size", 0),
|
||||
)
|
||||
replace_parameter(layer, params.WEIGHT, dg_weight)
|
||||
replace_parameter(layer, scale_attr, dg_weight_scale)
|
||||
replace_parameter(layer, params.block_scale_attr, dg_weight_scale)
|
||||
# bmm layers bypass this kernel and run through fp8_einsum.
|
||||
if not is_bmm:
|
||||
layer.deep_gemm_warmup_provider = self
|
||||
|
||||
def get_deep_gemm_warmup_weights(
|
||||
self, layer: torch.nn.Module
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
params = self._get_layer_params(layer)
|
||||
return params.weight, params.block_scale
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user