[Frontend][Build] Add vllm download-kernels for FlashInfer precompiled kernels (#58765)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Misha Goin
2026-10-09 21:22:35 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent be0823357f
commit 29e4d2d42e
15 changed files with 309 additions and 20 deletions
+53 -1
View File
@@ -8,6 +8,7 @@ Users of vLLM should always import **only** these wrappers.
import contextlib
import functools
import importlib
import importlib.metadata
import importlib.util
import os
import shutil
@@ -16,6 +17,7 @@ from typing import Any, NoReturn
import requests
import torch
from packaging.version import Version
import vllm.envs as envs
from vllm.logger import init_logger
@@ -98,12 +100,60 @@ def has_flashinfer() -> bool:
"FlashInfer kernels are disabled: flashinfer-cubin is not installed "
"and nvcc (CUDA_HOME, CUDA_PATH, PATH or /usr/local/cuda) or ninja "
"(PATH) is missing. Set CUDA_HOME to a CUDA toolkit and put ninja on "
"PATH, or run `flashinfer download-kernels`."
"PATH, or run `vllm download-kernels`."
)
return False
return True
def _installed_version(distribution: str) -> str | None:
try:
return Version(importlib.metadata.version(distribution)).public
except importlib.metadata.PackageNotFoundError:
return None
def has_flashinfer_jit_cache_wheels() -> bool:
"""Return whether FlashInfer publishes flashinfer-jit-cache for the installed
PyTorch's CUDA version, which it does from CUDA 12.9."""
cuda_version = torch.version.cuda
return cuda_version is not None and Version(cuda_version) >= Version("12.9")
def warn_if_flashinfer_kernels_missing() -> None:
"""Warn on Hopper and newer GPUs when FlashInfer's precompiled kernels are
missing or were installed for another FlashInfer version."""
if not (
current_platform.is_cuda()
and current_platform.has_device_capability(90)
and has_flashinfer()
):
return
flashinfer_version = _installed_version("flashinfer-python")
packages = []
if not envs.VLLM_HAS_FLASHINFER_CUBIN:
packages.append("flashinfer-cubin")
if has_flashinfer_jit_cache_wheels():
packages.append("flashinfer-jit-cache")
installed = {name: _installed_version(name) for name in packages}
if flashinfer_version is None or all(
version == flashinfer_version for version in installed.values()
):
return
logger.warning_once(
"FlashInfer's precompiled kernels are missing or do not match "
"flashinfer-python %s (%s). FlashInfer then downloads and compiles "
"kernels at startup, which can take several minutes, or fails to import "
"mismatched ones. Run `vllm download-kernels` in this Python environment "
"to install them.",
flashinfer_version,
", ".join(
f"{name} {version or 'not installed'}"
for name, version in installed.items()
),
)
@functools.cache
def has_flashinfer_bf16_gemm() -> bool:
"""Return whether FlashInfer exposes the BF16 dense GEMM API."""
@@ -1243,4 +1293,6 @@ __all__ = [
"should_use_flashinfer_for_blockscale_fp8_gemm",
"is_flashinfer_fp8_blockscale_gemm_supported",
"is_flashinfer_cudnn_fp8_prefill_attn_supported",
"has_flashinfer_jit_cache_wheels",
"warn_if_flashinfer_kernels_missing",
]