[Frontend][Build] Add vllm download-kernels for FlashInfer precompiled kernels (#58765)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
be0823357f
commit
29e4d2d42e
@@ -0,0 +1,87 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The `download-kernels` subcommand for the vLLM CLI.
|
||||
|
||||
Installs the precompiled kernels that vLLM's dependencies would otherwise
|
||||
download or compile at startup. Run it after installing or upgrading vLLM:
|
||||
|
||||
vllm download-kernels
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import typing
|
||||
from importlib.util import find_spec
|
||||
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _download_flashinfer_kernels(dry_run: bool) -> int:
|
||||
if find_spec("flashinfer") is None:
|
||||
logger.info("FlashInfer is not installed; skipping its kernels.")
|
||||
return 0
|
||||
from vllm.utils.flashinfer import has_flashinfer_jit_cache_wheels
|
||||
|
||||
cmd = [sys.executable, "-m", "flashinfer"]
|
||||
if has_flashinfer_jit_cache_wheels():
|
||||
cmd.append("download-kernels")
|
||||
else:
|
||||
logger.info(
|
||||
"FlashInfer does not publish flashinfer-jit-cache for this CUDA "
|
||||
"version; installing flashinfer-cubin only."
|
||||
)
|
||||
cmd.append("install-cubin-wheel")
|
||||
if dry_run:
|
||||
cmd.append("--dry-run")
|
||||
# FlashInfer refuses to import while kernels from another FlashInfer version
|
||||
# are installed, which would stop its own CLI from replacing them.
|
||||
env = {**os.environ, "FLASHINFER_DISABLE_VERSION_CHECK": "1"}
|
||||
return subprocess.run(cmd, env=env, check=False).returncode
|
||||
|
||||
|
||||
class DownloadKernelsSubcommand(CLISubcommand):
|
||||
"""The `download-kernels` subcommand for the vLLM CLI."""
|
||||
|
||||
name = "download-kernels"
|
||||
|
||||
@staticmethod
|
||||
def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Print the install commands without running them.",
|
||||
)
|
||||
return parser
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
sys.exit(_download_flashinfer_kernels(args.dry_run))
|
||||
|
||||
def subparser_init(
|
||||
self, subparsers: argparse._SubParsersAction
|
||||
) -> FlexibleArgumentParser:
|
||||
parser = subparsers.add_parser(
|
||||
self.name,
|
||||
help="Install precompiled kernels for faster startup.",
|
||||
description=(
|
||||
"Install the precompiled kernels that vLLM's dependencies would "
|
||||
"otherwise download or compile at startup. Run it after "
|
||||
"installing or upgrading vLLM."
|
||||
),
|
||||
usage="vllm download-kernels [--dry-run]",
|
||||
)
|
||||
return DownloadKernelsSubcommand.add_cli_args(parser)
|
||||
|
||||
|
||||
def cmd_init() -> list[CLISubcommand]:
|
||||
return [DownloadKernelsSubcommand()]
|
||||
@@ -42,6 +42,7 @@ def main():
|
||||
|
||||
import vllm.entrypoints.cli.benchmark.main
|
||||
import vllm.entrypoints.cli.collect_env
|
||||
import vllm.entrypoints.cli.download_kernels
|
||||
import vllm.entrypoints.cli.launch
|
||||
import vllm.entrypoints.cli.openai
|
||||
import vllm.entrypoints.cli.preload
|
||||
@@ -60,6 +61,7 @@ def main():
|
||||
vllm.entrypoints.cli.launch,
|
||||
vllm.entrypoints.cli.benchmark.main,
|
||||
vllm.entrypoints.cli.collect_env,
|
||||
vllm.entrypoints.cli.download_kernels,
|
||||
vllm.entrypoints.cli.preload,
|
||||
vllm.entrypoints.cli.run_batch,
|
||||
vllm.entrypoints.cli.snapshot,
|
||||
|
||||
@@ -8,6 +8,7 @@ Users of vLLM should always import **only** these wrappers.
|
||||
import contextlib
|
||||
import functools
|
||||
import importlib
|
||||
import importlib.metadata
|
||||
import importlib.util
|
||||
import os
|
||||
import shutil
|
||||
@@ -16,6 +17,7 @@ from typing import Any, NoReturn
|
||||
|
||||
import requests
|
||||
import torch
|
||||
from packaging.version import Version
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
@@ -98,12 +100,60 @@ def has_flashinfer() -> bool:
|
||||
"FlashInfer kernels are disabled: flashinfer-cubin is not installed "
|
||||
"and nvcc (CUDA_HOME, CUDA_PATH, PATH or /usr/local/cuda) or ninja "
|
||||
"(PATH) is missing. Set CUDA_HOME to a CUDA toolkit and put ninja on "
|
||||
"PATH, or run `flashinfer download-kernels`."
|
||||
"PATH, or run `vllm download-kernels`."
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _installed_version(distribution: str) -> str | None:
|
||||
try:
|
||||
return Version(importlib.metadata.version(distribution)).public
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def has_flashinfer_jit_cache_wheels() -> bool:
|
||||
"""Return whether FlashInfer publishes flashinfer-jit-cache for the installed
|
||||
PyTorch's CUDA version, which it does from CUDA 12.9."""
|
||||
cuda_version = torch.version.cuda
|
||||
return cuda_version is not None and Version(cuda_version) >= Version("12.9")
|
||||
|
||||
|
||||
def warn_if_flashinfer_kernels_missing() -> None:
|
||||
"""Warn on Hopper and newer GPUs when FlashInfer's precompiled kernels are
|
||||
missing or were installed for another FlashInfer version."""
|
||||
if not (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.has_device_capability(90)
|
||||
and has_flashinfer()
|
||||
):
|
||||
return
|
||||
flashinfer_version = _installed_version("flashinfer-python")
|
||||
packages = []
|
||||
if not envs.VLLM_HAS_FLASHINFER_CUBIN:
|
||||
packages.append("flashinfer-cubin")
|
||||
if has_flashinfer_jit_cache_wheels():
|
||||
packages.append("flashinfer-jit-cache")
|
||||
installed = {name: _installed_version(name) for name in packages}
|
||||
if flashinfer_version is None or all(
|
||||
version == flashinfer_version for version in installed.values()
|
||||
):
|
||||
return
|
||||
logger.warning_once(
|
||||
"FlashInfer's precompiled kernels are missing or do not match "
|
||||
"flashinfer-python %s (%s). FlashInfer then downloads and compiles "
|
||||
"kernels at startup, which can take several minutes, or fails to import "
|
||||
"mismatched ones. Run `vllm download-kernels` in this Python environment "
|
||||
"to install them.",
|
||||
flashinfer_version,
|
||||
", ".join(
|
||||
f"{name} {version or 'not installed'}"
|
||||
for name, version in installed.items()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_bf16_gemm() -> bool:
|
||||
"""Return whether FlashInfer exposes the BF16 dense GEMM API."""
|
||||
@@ -1243,4 +1293,6 @@ __all__ = [
|
||||
"should_use_flashinfer_for_blockscale_fp8_gemm",
|
||||
"is_flashinfer_fp8_blockscale_gemm_supported",
|
||||
"is_flashinfer_cudnn_fp8_prefill_attn_supported",
|
||||
"has_flashinfer_jit_cache_wheels",
|
||||
"warn_if_flashinfer_kernels_missing",
|
||||
]
|
||||
|
||||
@@ -73,6 +73,7 @@ from vllm.profiler.wrapper import (
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.flashinfer import warn_if_flashinfer_kernels_missing
|
||||
from vllm.utils.gc_utils import freeze_gc_heap, maybe_attach_gc_debug_callback
|
||||
from vllm.utils.gpu_sync_debug import enable_gpu_sync_check, with_gpu_sync_check
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
@@ -497,6 +498,7 @@ class Worker(WorkerBase):
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
logger.info_once("Using V2 Model Runner")
|
||||
warn_if_flashinfer_kernels_missing()
|
||||
|
||||
# Set random seed.
|
||||
set_random_seed(self.model_config.seed)
|
||||
|
||||
Reference in New Issue
Block a user