[CT][FP8][Marlin] refactor CompressedTensorsW8A16Fp8 to use kernel abstraction (#38244)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Signed-off-by: Kunshang Ji <jikunshang95@gmail.com>
This commit is contained in:
@@ -177,6 +177,21 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
||||
],
|
||||
}
|
||||
|
||||
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
|
||||
PlatformEnum.CUDA: [
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
# To be added
|
||||
],
|
||||
PlatformEnum.CPU: [
|
||||
# To be added
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
# To be added
|
||||
],
|
||||
}
|
||||
|
||||
# in priority/performance order (when available)
|
||||
_POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
|
||||
PlatformEnum.CUDA: [
|
||||
@@ -463,6 +478,41 @@ def choose_mp_linear_kernel(
|
||||
)
|
||||
|
||||
|
||||
def init_wfp8_a16_linear_kernel(
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey,
|
||||
weight_shape: tuple[int, int],
|
||||
input_dtype: torch.dtype,
|
||||
out_dtype: torch.dtype,
|
||||
force_kernel: type[FP8ScaledMMLinearKernel] | None = None,
|
||||
module_name: str | None = None,
|
||||
) -> FP8ScaledMMLinearKernel:
|
||||
config = FP8ScaledMMLinearLayerConfig(
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
weight_shape=weight_shape,
|
||||
input_dtype=input_dtype,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
kernel_type = choose_scaled_mm_linear_kernel(
|
||||
config, _POSSIBLE_WFP8A16_KERNELS, force_kernel=force_kernel
|
||||
)
|
||||
|
||||
if module_name:
|
||||
logger.info_once(
|
||||
"Selected %s for %s",
|
||||
kernel_type.__name__,
|
||||
module_name,
|
||||
scope="global",
|
||||
)
|
||||
|
||||
return kernel_type(
|
||||
config,
|
||||
layer_param_names=["weight", "weight_scale", "input_scale", "input_scale_ub"],
|
||||
)
|
||||
|
||||
|
||||
# Maps VLLM_NVFP4_GEMM_BACKEND env var values to kernel classes.
|
||||
_NVFP4_BACKEND_TO_KERNEL: dict[str, type[NvFp4LinearKernel]] = {
|
||||
"flashinfer-cutlass": FlashInferCutlassNvFp4LinearKernel,
|
||||
@@ -588,6 +638,7 @@ __all__ = [
|
||||
"init_nvfp4_linear_kernel",
|
||||
"choose_mp_linear_kernel",
|
||||
"register_linear_kernel",
|
||||
"init_wfp8_a16_linear_kernel",
|
||||
"FP8ScaledMMLinearKernel",
|
||||
"Int8ScaledMMLinearKernel",
|
||||
"ScaledMMLinearKernel",
|
||||
|
||||
Reference in New Issue
Block a user