[Quant] Make static quant support all group shapes (#30833)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
+13
-4
@@ -1752,6 +1752,7 @@ def scaled_fp8_quant(
|
||||
scale_ub: torch.Tensor | None = None,
|
||||
use_per_token_if_dynamic: bool = False,
|
||||
output: torch.Tensor | None = None,
|
||||
group_shape: tuple[int, int] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Quantize input tensor to FP8 and return quantized tensor and scale.
|
||||
@@ -1763,14 +1764,23 @@ def scaled_fp8_quant(
|
||||
will benefit from padding.
|
||||
|
||||
Args:
|
||||
input: The input tensor to be quantized to FP8
|
||||
scale: Optional scaling factor for the FP8 quantization
|
||||
input: The input tensor to be quantized to FP8 (must be 2D: [M, N])
|
||||
scale: Optional scaling factor for the FP8 quantization. Supports:
|
||||
- 0D or [1]: per-tensor scaling
|
||||
- 1D: requires explicit group_shape to disambiguate per-channel
|
||||
vs per-token (use (-1, 1) for per-channel, (1, -1) for per-token)
|
||||
- 2D [M/group_m, N/group_n]: group scaling (e.g. [M, N/128] for
|
||||
DeepSeek-style (1,128) groups, or [M/128, N/128] for (128,128))
|
||||
scale_ub: Optional upper bound for scaling factor in dynamic
|
||||
per token case
|
||||
num_token_padding: If specified, pad the first dimension
|
||||
of the output to at least this value.
|
||||
use_per_token_if_dynamic: Whether to do per_tensor or per_token
|
||||
in the dynamic quantization case.
|
||||
group_shape: Optional tuple (group_m, group_n) specifying the group
|
||||
shape for static quantization. Use -1 for "full extent" (e.g.,
|
||||
(-1, -1) for per-tensor, (-1, 1) for per-channel, etc.)
|
||||
Required for 1D scales; optional for 2D scales.
|
||||
|
||||
Returns:
|
||||
tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and
|
||||
@@ -1799,8 +1809,7 @@ def scaled_fp8_quant(
|
||||
scale = torch.empty(1, device=input.device, dtype=torch.float32)
|
||||
torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
|
||||
else:
|
||||
assert scale.numel() == 1, f"{scale.shape}"
|
||||
torch.ops._C.static_scaled_fp8_quant(output, input, scale)
|
||||
torch.ops._C.static_scaled_fp8_quant(output, input, scale, group_shape)
|
||||
|
||||
return output, scale
|
||||
|
||||
|
||||
Reference in New Issue
Block a user