[Kernel] fix types used in aqlm and ggml kernels to support dynamo (#7596)
This commit is contained in:
+7
-21
@@ -17,13 +17,7 @@ if not current_platform.is_tpu():
|
||||
logger.warning("Failed to import from vllm._C with %r", e)
|
||||
|
||||
with contextlib.suppress(ImportError):
|
||||
# ruff: noqa: F401
|
||||
import vllm._moe_C
|
||||
|
||||
|
||||
def is_custom_op_supported(op_name: str) -> bool:
|
||||
op, overloads = torch._C._jit_get_operation(op_name)
|
||||
return op is not None
|
||||
import vllm._moe_C # noqa: F401
|
||||
|
||||
|
||||
def hint_on_error(fn):
|
||||
@@ -280,14 +274,14 @@ def cutlass_scaled_mm_azp(a: torch.Tensor,
|
||||
# aqlm
|
||||
def aqlm_gemm(input: torch.Tensor, codes: torch.Tensor,
|
||||
codebooks: torch.Tensor, scales: torch.Tensor,
|
||||
codebook_partition_sizes: torch.Tensor,
|
||||
codebook_partition_sizes: List[int],
|
||||
bias: Optional[torch.Tensor]) -> torch.Tensor:
|
||||
return torch.ops._C.aqlm_gemm(input, codes, codebooks, scales,
|
||||
codebook_partition_sizes, bias)
|
||||
|
||||
|
||||
def aqlm_dequant(codes: torch.Tensor, codebooks: torch.Tensor,
|
||||
codebook_partition_sizes: torch.Tensor) -> torch.Tensor:
|
||||
codebook_partition_sizes: List[int]) -> torch.Tensor:
|
||||
return torch.ops._C.aqlm_dequant(codes, codebooks,
|
||||
codebook_partition_sizes)
|
||||
|
||||
@@ -434,25 +428,17 @@ def marlin_qqq_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
|
||||
|
||||
|
||||
# gguf
|
||||
def ggml_dequantize(W: torch.Tensor, quant_type: int, m: int, n: int):
|
||||
def ggml_dequantize(W: torch.Tensor, quant_type: int, m: int,
|
||||
n: int) -> torch.Tensor:
|
||||
return torch.ops._C.ggml_dequantize(W, quant_type, m, n)
|
||||
|
||||
|
||||
def ggml_mul_mat_vec(
|
||||
W: torch.Tensor,
|
||||
X: torch.Tensor,
|
||||
quant_type: int,
|
||||
row: int,
|
||||
):
|
||||
return torch.ops._C.ggml_mul_mat_vec(W, X, quant_type, row)
|
||||
|
||||
|
||||
def ggml_mul_mat_vec_a8(
|
||||
W: torch.Tensor,
|
||||
X: torch.Tensor,
|
||||
quant_type: int,
|
||||
row: int,
|
||||
):
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.ggml_mul_mat_vec_a8(W, X, quant_type, row)
|
||||
|
||||
|
||||
@@ -461,7 +447,7 @@ def ggml_mul_mat_a8(
|
||||
X: torch.Tensor,
|
||||
quant_type: int,
|
||||
row: int,
|
||||
):
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.ggml_mul_mat_a8(W, X, quant_type, row)
|
||||
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ def generic_dequantize_gemm(
|
||||
codebooks: torch.
|
||||
Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
|
||||
scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
|
||||
output_partition_sizes: torch.IntTensor,
|
||||
output_partition_sizes: List[int],
|
||||
bias: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
output_shape = input.shape[:-1] + (scales.shape[0], )
|
||||
@@ -133,7 +133,7 @@ def optimized_dequantize_gemm(
|
||||
codebooks: torch.
|
||||
Tensor, # [num_codebooks, codebook_size, out_group_size, in_group_size]
|
||||
scales: torch.Tensor, # [num_out_groups, 1, 1, 1]
|
||||
output_partition_sizes: torch.IntTensor,
|
||||
output_partition_sizes: List[int],
|
||||
bias: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
weights = ops.aqlm_dequant(codes, codebooks, output_partition_sizes)
|
||||
@@ -288,10 +288,8 @@ class AQLMLinearMethod(LinearMethodBase):
|
||||
codebooks,
|
||||
{
|
||||
# metadata indicates fixed size concatenated along dim 0
|
||||
"is_metadata":
|
||||
True,
|
||||
"output_partition_sizes":
|
||||
torch.tensor(output_partition_sizes, device='cpu'),
|
||||
"is_metadata": True,
|
||||
"output_partition_sizes": output_partition_sizes
|
||||
},
|
||||
)
|
||||
|
||||
@@ -334,7 +332,7 @@ class AQLMLinearMethod(LinearMethodBase):
|
||||
codes = layer.codes
|
||||
scales = layer.scales
|
||||
output_partition_sizes = getattr(codebooks, "output_partition_sizes",
|
||||
None)
|
||||
[])
|
||||
|
||||
nbooks = codes.shape[2]
|
||||
ingroups = codebooks.shape[3]
|
||||
|
||||
@@ -212,6 +212,7 @@ class GPTQLinearMethod(LinearMethodBase):
|
||||
layer.g_idx.data = torch.argsort(layer.g_idx).to(torch.int)
|
||||
else:
|
||||
layer.g_idx.data = torch.empty((0, ),
|
||||
dtype=torch.int,
|
||||
device=layer.g_idx.device)
|
||||
layer.exllama_state = ExllamaState.READY
|
||||
ops.gptq_shuffle(layer.qweight, layer.g_idx,
|
||||
|
||||
Reference in New Issue
Block a user