Enable group size 64 for Machete (#20290)
Signed-off-by: czhu-cohere <conway.zhu@cohere.com>
This commit is contained in:
@@ -7,7 +7,6 @@ import torch
|
||||
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
|
||||
MACHETE_SUPPORTED_GROUP_SIZES = [-1, 128]
|
||||
MACHETE_PREPACKED_BLOCK_SHAPE = [64, 128]
|
||||
|
||||
|
||||
@@ -22,6 +21,24 @@ def query_machete_supported_act_types(zero_points: bool) -> list[ScalarType]:
|
||||
return [torch.float16, torch.bfloat16]
|
||||
|
||||
|
||||
def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
|
||||
"""
|
||||
Queries the supported group sizes for Machete based on the activation type.
|
||||
|
||||
Args:
|
||||
act_type: The activation data type (torch.float16, torch.bfloat16).
|
||||
|
||||
Returns:
|
||||
A list of supported group sizes. The group size must
|
||||
be divisible by `TileShapeK = 128 * 8 // num_bits(act_type)`.
|
||||
-1 indicates per-channel quantization.
|
||||
"""
|
||||
if act_type in [torch.float16, torch.bfloat16]:
|
||||
return [-1, 64, 128]
|
||||
else:
|
||||
return [-1, 128]
|
||||
|
||||
|
||||
def check_machete_supports_shape(in_features: int, out_featrues: int) \
|
||||
-> tuple[bool, Optional[str]]:
|
||||
if in_features % MACHETE_PREPACKED_BLOCK_SHAPE[0] != 0:
|
||||
|
||||
Reference in New Issue
Block a user