[Frontend][Build] Add vllm download-kernels for FlashInfer precompiled kernels (#58765)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Misha Goin
2026-10-09 21:22:35 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent be0823357f
commit 29e4d2d42e
15 changed files with 309 additions and 20 deletions
+87
View File
@@ -0,0 +1,87 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The `download-kernels` subcommand for the vLLM CLI.
Installs the precompiled kernels that vLLM's dependencies would otherwise
download or compile at startup. Run it after installing or upgrading vLLM:
vllm download-kernels
"""
import argparse
import os
import subprocess
import sys
import typing
from importlib.util import find_spec
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.logger import init_logger
if typing.TYPE_CHECKING:
from vllm.utils.argparse_utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
logger = init_logger(__name__)
def _download_flashinfer_kernels(dry_run: bool) -> int:
if find_spec("flashinfer") is None:
logger.info("FlashInfer is not installed; skipping its kernels.")
return 0
from vllm.utils.flashinfer import has_flashinfer_jit_cache_wheels
cmd = [sys.executable, "-m", "flashinfer"]
if has_flashinfer_jit_cache_wheels():
cmd.append("download-kernels")
else:
logger.info(
"FlashInfer does not publish flashinfer-jit-cache for this CUDA "
"version; installing flashinfer-cubin only."
)
cmd.append("install-cubin-wheel")
if dry_run:
cmd.append("--dry-run")
# FlashInfer refuses to import while kernels from another FlashInfer version
# are installed, which would stop its own CLI from replacing them.
env = {**os.environ, "FLASHINFER_DISABLE_VERSION_CHECK": "1"}
return subprocess.run(cmd, env=env, check=False).returncode
class DownloadKernelsSubcommand(CLISubcommand):
"""The `download-kernels` subcommand for the vLLM CLI."""
name = "download-kernels"
@staticmethod
def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument(
"--dry-run",
action="store_true",
help="Print the install commands without running them.",
)
return parser
@staticmethod
def cmd(args: argparse.Namespace) -> None:
sys.exit(_download_flashinfer_kernels(args.dry_run))
def subparser_init(
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
parser = subparsers.add_parser(
self.name,
help="Install precompiled kernels for faster startup.",
description=(
"Install the precompiled kernels that vLLM's dependencies would "
"otherwise download or compile at startup. Run it after "
"installing or upgrading vLLM."
),
usage="vllm download-kernels [--dry-run]",
)
return DownloadKernelsSubcommand.add_cli_args(parser)
def cmd_init() -> list[CLISubcommand]:
return [DownloadKernelsSubcommand()]
+2
View File
@@ -42,6 +42,7 @@ def main():
import vllm.entrypoints.cli.benchmark.main
import vllm.entrypoints.cli.collect_env
import vllm.entrypoints.cli.download_kernels
import vllm.entrypoints.cli.launch
import vllm.entrypoints.cli.openai
import vllm.entrypoints.cli.preload
@@ -60,6 +61,7 @@ def main():
vllm.entrypoints.cli.launch,
vllm.entrypoints.cli.benchmark.main,
vllm.entrypoints.cli.collect_env,
vllm.entrypoints.cli.download_kernels,
vllm.entrypoints.cli.preload,
vllm.entrypoints.cli.run_batch,
vllm.entrypoints.cli.snapshot,
+53 -1
View File
@@ -8,6 +8,7 @@ Users of vLLM should always import **only** these wrappers.
import contextlib
import functools
import importlib
import importlib.metadata
import importlib.util
import os
import shutil
@@ -16,6 +17,7 @@ from typing import Any, NoReturn
import requests
import torch
from packaging.version import Version
import vllm.envs as envs
from vllm.logger import init_logger
@@ -98,12 +100,60 @@ def has_flashinfer() -> bool:
"FlashInfer kernels are disabled: flashinfer-cubin is not installed "
"and nvcc (CUDA_HOME, CUDA_PATH, PATH or /usr/local/cuda) or ninja "
"(PATH) is missing. Set CUDA_HOME to a CUDA toolkit and put ninja on "
"PATH, or run `flashinfer download-kernels`."
"PATH, or run `vllm download-kernels`."
)
return False
return True
def _installed_version(distribution: str) -> str | None:
try:
return Version(importlib.metadata.version(distribution)).public
except importlib.metadata.PackageNotFoundError:
return None
def has_flashinfer_jit_cache_wheels() -> bool:
"""Return whether FlashInfer publishes flashinfer-jit-cache for the installed
PyTorch's CUDA version, which it does from CUDA 12.9."""
cuda_version = torch.version.cuda
return cuda_version is not None and Version(cuda_version) >= Version("12.9")
def warn_if_flashinfer_kernels_missing() -> None:
"""Warn on Hopper and newer GPUs when FlashInfer's precompiled kernels are
missing or were installed for another FlashInfer version."""
if not (
current_platform.is_cuda()
and current_platform.has_device_capability(90)
and has_flashinfer()
):
return
flashinfer_version = _installed_version("flashinfer-python")
packages = []
if not envs.VLLM_HAS_FLASHINFER_CUBIN:
packages.append("flashinfer-cubin")
if has_flashinfer_jit_cache_wheels():
packages.append("flashinfer-jit-cache")
installed = {name: _installed_version(name) for name in packages}
if flashinfer_version is None or all(
version == flashinfer_version for version in installed.values()
):
return
logger.warning_once(
"FlashInfer's precompiled kernels are missing or do not match "
"flashinfer-python %s (%s). FlashInfer then downloads and compiles "
"kernels at startup, which can take several minutes, or fails to import "
"mismatched ones. Run `vllm download-kernels` in this Python environment "
"to install them.",
flashinfer_version,
", ".join(
f"{name} {version or 'not installed'}"
for name, version in installed.items()
),
)
@functools.cache
def has_flashinfer_bf16_gemm() -> bool:
"""Return whether FlashInfer exposes the BF16 dense GEMM API."""
@@ -1243,4 +1293,6 @@ __all__ = [
"should_use_flashinfer_for_blockscale_fp8_gemm",
"is_flashinfer_fp8_blockscale_gemm_supported",
"is_flashinfer_cudnn_fp8_prefill_attn_supported",
"has_flashinfer_jit_cache_wheels",
"warn_if_flashinfer_kernels_missing",
]
+2
View File
@@ -73,6 +73,7 @@ from vllm.profiler.wrapper import (
from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
from vllm.tracing import instrument
from vllm.utils.flashinfer import warn_if_flashinfer_kernels_missing
from vllm.utils.gc_utils import freeze_gc_heap, maybe_attach_gc_debug_callback
from vllm.utils.gpu_sync_debug import enable_gpu_sync_check, with_gpu_sync_check
from vllm.utils.mem_constants import GiB_bytes
@@ -497,6 +498,7 @@ class Worker(WorkerBase):
if self.use_v2_model_runner:
logger.info_once("Using V2 Model Runner")
warn_if_flashinfer_kernels_missing()
# Set random seed.
set_random_seed(self.model_config.seed)