[Kernel][UX] Add --linear-backend arg for linear kernel selection (#39538)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-05-15 17:07:39 -07:00
committed by GitHub
parent d0921bafef
commit b2a27b82d9
4 changed files with 266 additions and 34 deletions
+44
View File
@@ -135,6 +135,24 @@ MoEBackend = Literal[
"emulation",
]
LinearBackend = Literal[
"auto",
"cutlass",
"flashinfer_cutlass",
"flashinfer_trtllm",
"flashinfer_cudnn",
"marlin",
"triton",
"deep_gemm",
"torch",
"aiter",
"machete",
"fbgemm",
"conch",
"exllama",
"emulation",
]
@config
class KernelConfig:
@@ -168,6 +186,25 @@ class KernelConfig:
running QDQ on activations.
"""
linear_backend: LinearBackend = "auto"
"""Backend for quantized linear layer GEMM kernels. Available options:
- "auto": Automatically select the best backend based on model and hardware
- "cutlass": Use CUTLASS-based kernels
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
- "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels
- "flashinfer_cudnn": Use FlashInfer with cuDNN kernels
- "marlin": Use Marlin kernels
- "triton": Use Triton-based kernels
- "deep_gemm": Use DeepGEMM kernels
- "torch": Use PyTorch native scaled_mm kernels
- "aiter": Use AMD AITer kernels (ROCm only)
- "machete": Use Machete kernels (mixed-precision)
- "fbgemm": Use FBGEMM kernels
- "conch": Use Conch mixed-precision kernels
- "exllama": Use Exllama mixed-precision kernels
- "emulation": Use slow dequant-to-BF16 emulation (for testing only)"""
@field_validator("moe_backend", mode="before")
@classmethod
def _normalize_moe_backend(cls, value: Any) -> Any:
@@ -175,6 +212,13 @@ class KernelConfig:
return value.lower().replace("-", "_")
return value
@field_validator("linear_backend", mode="before")
@classmethod
def _normalize_linear_backend(cls, value: Any) -> Any:
if isinstance(value, str):
return value.lower().replace("-", "_")
return value
def compute_hash(self) -> str:
"""
Produces a hash unique to the pass configuration.