Enable group size 64 for Machete (#20290)

Signed-off-by: czhu-cohere <conway.zhu@cohere.com>
This commit is contained in:
czhu-cohere
2025-07-01 18:05:44 -07:00
committed by GitHub
parent e81fbefe8a
commit 3abfe22154
3 changed files with 25 additions and 8 deletions
@@ -7,7 +7,6 @@ import torch
from vllm.scalar_type import ScalarType, scalar_types
MACHETE_SUPPORTED_GROUP_SIZES = [-1, 128]
MACHETE_PREPACKED_BLOCK_SHAPE = [64, 128]
@@ -22,6 +21,24 @@ def query_machete_supported_act_types(zero_points: bool) -> list[ScalarType]:
return [torch.float16, torch.bfloat16]
def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
"""
Queries the supported group sizes for Machete based on the activation type.
Args:
act_type: The activation data type (torch.float16, torch.bfloat16).
Returns:
A list of supported group sizes. The group size must
be divisible by `TileShapeK = 128 * 8 // num_bits(act_type)`.
-1 indicates per-channel quantization.
"""
if act_type in [torch.float16, torch.bfloat16]:
return [-1, 64, 128]
else:
return [-1, 128]
def check_machete_supports_shape(in_features: int, out_featrues: int) \
-> tuple[bool, Optional[str]]:
if in_features % MACHETE_PREPACKED_BLOCK_SHAPE[0] != 0: