[Kernel] GGUF MMVQ kernel for multiple input vectors (#18754)

Signed-off-by: SzymonOzog <szymon.ozog@gmail.com>
This commit is contained in:
Szymon Ożóg
2025-06-16 17:33:26 +08:00
committed by GitHub
parent 8d120701fd
commit dec66d253b
4 changed files with 95 additions and 87 deletions
+1 -1
View File
@@ -594,7 +594,7 @@ if hasattr(torch.ops._C, "ggml_dequantize"):
quant_type: int,
row: torch.SymInt,
) -> torch.Tensor:
return torch.empty((1, row), dtype=X.dtype, device=W.device)
return torch.empty((X.shape[0], row), dtype=X.dtype, device=W.device)
@register_fake("_C::ggml_mul_mat_a8")
def _ggml_mul_mat_a8_fake(