[Refactor][Quantization] Make FP8 and MLA weight transforms reusable pure functions (#57732)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -1373,7 +1373,7 @@ def process_fp8_weight_tensor_strategy(
|
||||
logical_widths: list[int],
|
||||
input_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""Process weights for tensor-wise quantization strategy."""
|
||||
"""Requantize fused shards to one scale and return ``(K, N)`` weight."""
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
requantize_with_max_scale,
|
||||
@@ -1391,7 +1391,7 @@ def process_fp8_weight_tensor_strategy(
|
||||
logical_widths=logical_widths,
|
||||
)
|
||||
|
||||
weight = _maybe_pad_fp8_weight(weight)
|
||||
weight = _maybe_pad_fp8_weight(weight).t()
|
||||
return weight, weight_scale, input_scale
|
||||
|
||||
|
||||
@@ -1400,7 +1400,7 @@ def process_fp8_weight_channel_strategy(
|
||||
weight_scale: torch.Tensor,
|
||||
input_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""Process weights for channel-wise quantization strategy."""
|
||||
"""Normalize FNUZ if needed and return ``(K, N)`` weight."""
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
)
|
||||
@@ -1410,14 +1410,14 @@ def process_fp8_weight_channel_strategy(
|
||||
weight=weight, weight_scale=weight_scale, input_scale=input_scale
|
||||
)
|
||||
|
||||
return weight, weight_scale, input_scale
|
||||
return weight.t(), weight_scale, input_scale
|
||||
|
||||
|
||||
def process_fp8_weight_block_strategy(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Process weights for block-wise quantization strategy."""
|
||||
"""Normalize FNUZ if needed and return ``(N, K)`` weight (no transpose)."""
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user