[Kernel][UX] Add --linear-backend arg for linear kernel selection (#39538)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -135,6 +135,24 @@ MoEBackend = Literal[
|
||||
"emulation",
|
||||
]
|
||||
|
||||
LinearBackend = Literal[
|
||||
"auto",
|
||||
"cutlass",
|
||||
"flashinfer_cutlass",
|
||||
"flashinfer_trtllm",
|
||||
"flashinfer_cudnn",
|
||||
"marlin",
|
||||
"triton",
|
||||
"deep_gemm",
|
||||
"torch",
|
||||
"aiter",
|
||||
"machete",
|
||||
"fbgemm",
|
||||
"conch",
|
||||
"exllama",
|
||||
"emulation",
|
||||
]
|
||||
|
||||
|
||||
@config
|
||||
class KernelConfig:
|
||||
@@ -168,6 +186,25 @@ class KernelConfig:
|
||||
running QDQ on activations.
|
||||
"""
|
||||
|
||||
linear_backend: LinearBackend = "auto"
|
||||
"""Backend for quantized linear layer GEMM kernels. Available options:
|
||||
|
||||
- "auto": Automatically select the best backend based on model and hardware
|
||||
- "cutlass": Use CUTLASS-based kernels
|
||||
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
|
||||
- "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels
|
||||
- "flashinfer_cudnn": Use FlashInfer with cuDNN kernels
|
||||
- "marlin": Use Marlin kernels
|
||||
- "triton": Use Triton-based kernels
|
||||
- "deep_gemm": Use DeepGEMM kernels
|
||||
- "torch": Use PyTorch native scaled_mm kernels
|
||||
- "aiter": Use AMD AITer kernels (ROCm only)
|
||||
- "machete": Use Machete kernels (mixed-precision)
|
||||
- "fbgemm": Use FBGEMM kernels
|
||||
- "conch": Use Conch mixed-precision kernels
|
||||
- "exllama": Use Exllama mixed-precision kernels
|
||||
- "emulation": Use slow dequant-to-BF16 emulation (for testing only)"""
|
||||
|
||||
@field_validator("moe_backend", mode="before")
|
||||
@classmethod
|
||||
def _normalize_moe_backend(cls, value: Any) -> Any:
|
||||
@@ -175,6 +212,13 @@ class KernelConfig:
|
||||
return value.lower().replace("-", "_")
|
||||
return value
|
||||
|
||||
@field_validator("linear_backend", mode="before")
|
||||
@classmethod
|
||||
def _normalize_linear_backend(cls, value: Any) -> Any:
|
||||
if isinstance(value, str):
|
||||
return value.lower().replace("-", "_")
|
||||
return value
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Produces a hash unique to the pass configuration.
|
||||
|
||||
Reference in New Issue
Block a user