diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 6cbb65e26d..5d513f767f 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -186,12 +186,13 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = # in priority/performance order (when available) _POSSIBLE_FP8_BLOCK_KERNELS: dict[ - PlatformEnum, list[type[Fp8BlockScaledMMLinearKernel]] + PlatformEnum, list[type[Fp8BlockScaledMMLinearKernel | FP8ScaledMMLinearKernel]] ] = { PlatformEnum.CUDA: [ FlashInferFp8DeepGEMMDynamicBlockScaledKernel, DeepGemmFp8BlockScaledMMKernel, CutlassFp8BlockScaledMMKernel, + MarlinFP8ScaledMMLinearKernel, TritonFp8BlockScaledMMKernel, ], PlatformEnum.ROCM: [ @@ -392,6 +393,19 @@ def init_fp8_linear_kernel( scope="global", ) + # TODO make scaled_mm kernels inherit from MMLinearKernel + # only MarlinFP8ScaledMMLinearKernel is a type of FP8ScaledMMLinearKernel. + if issubclass(kernel_type, FP8ScaledMMLinearKernel): + return kernel_type( + scaled_mm_linear_kernel_config, + layer_param_names=[ + "weight", + "weight_scale", + "input_scale", + "input_scale_ub", + ], + ) + return kernel_type( scaled_mm_linear_kernel_config, ) @@ -399,7 +413,7 @@ def init_fp8_linear_kernel( else: kernel_type = choose_scaled_mm_linear_kernel( config=scaled_mm_linear_kernel_config, - possible_kernels=_POSSIBLE_FP8_KERNELS, # type: ignore[misc] + possible_kernels=_POSSIBLE_FP8_KERNELS, # type: ignore[arg-type] force_kernel=force_kernel, ) if module_name: