iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
sm86
vLLM/setup.py
T
MOVZXandHinata 5f3c7bd8ab [Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests 
- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm
  target path; drop the rocm requirement files
- Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules
  and retarget the linear kernel registries away from them
- Reduce platforms/rocm.py to the GCN architecture predicates that live
  code still imports, and delete the RocmPlatform class and plugin
- Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc,
  server_info and sleep routes plus the log_error_stack flag
- Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins;
  the live server process maps none of them
- Delete tests that only build architectures this fork no longer
  registers, and test directories left without any test file
- Restore the tests/v1/kv_connector mock helpers that scheduler tests
  import at module level

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
2026-10-19 19:31:07 +07:00

1095 lines
42 KiB
Python
Raw Permalink Blame History

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import sysconfig
from pathlib import Path
from shutil import which
import torch
from packaging.version import Version, parse
from setuptools import Extension, setup
from setuptools.command.build_ext import build_ext
from setuptools_scm import get_version
from torch.utils.cpp_extension import CUDA_HOME
# Select vLLM release tags, excluding crate tags such as "proto-v0.3.0".
VLLM_GIT_DESCRIBE_COMMAND = [
"git",
"describe",
"--dirty",
"--tags",
"--long",
"--abbrev=40",
"--match",
"v[0-9]*",
]
def load_module_from_path(module_name, path):
spec = importlib.util.spec_from_file_location(module_name, path)
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
return module
ROOT_DIR = Path(__file__).parent
logger = logging.getLogger(__name__)
# cannot import envs directly because it depends on vllm,
# which is not installed yet
envs = load_module_from_path("envs", os.path.join(ROOT_DIR, "vllm", "envs.py"))
VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
USE_PRECOMPILED_EXTENSIONS = envs.VLLM_USE_PRECOMPILED
def is_metadata_only_build() -> bool:
return bool({"egg_info", "dist_info"}.intersection(sys.argv[1:]))
if sys.platform.startswith("darwin"):
logger.warning(
"This build only supports CUDA; building without GPU extensions"
)
VLLM_TARGET_DEVICE = "empty"
elif not (sys.platform.startswith("linux") or sys.platform.startswith("darwin")):
logger.warning(
"vLLM only supports Linux platform (including WSL) and MacOS."
"Building on %s, "
"so vLLM may not be able to run correctly",
sys.platform,
)
VLLM_TARGET_DEVICE = "empty"
elif sys.platform.startswith("linux") and os.getenv("VLLM_TARGET_DEVICE") is None:
if torch.version.xpu is not None:
VLLM_TARGET_DEVICE = "xpu"
logger.info("Auto-detected XPU")
elif torch.version.cuda is not None:
VLLM_TARGET_DEVICE = "cuda"
logger.info("Auto-detected CUDA")
else:
VLLM_TARGET_DEVICE = "empty"
if VLLM_TARGET_DEVICE == "rocm":
VLLM_TARGET_DEVICE = "empty"
def is_sccache_available() -> bool:
return which("sccache") is not None and not bool(
int(os.getenv("VLLM_DISABLE_SCCACHE", "0"))
)
def is_ccache_available() -> bool:
return which("ccache") is not None
def is_ninja_available() -> bool:
return which("ninja") is not None
def is_freethreaded():
return bool(sysconfig.get_config_var("Py_GIL_DISABLED"))
class CMakeExtension(Extension):
def __init__(self, name: str, cmake_lists_dir: str = ".", **kwa) -> None:
super().__init__(name, sources=[], py_limited_api=not is_freethreaded(), **kwa)
self.cmake_lists_dir = os.path.abspath(cmake_lists_dir)
class cmake_build_ext(build_ext):
# A dict of extension directories that have been configured.
did_config: dict[str, bool] = {}
#
# Determine number of compilation jobs and optionally nvcc compile threads.
#
def compute_num_jobs(self):
# `num_jobs` is either the value of the MAX_JOBS environment variable
# (if defined) or the number of CPUs available.
num_jobs = envs.MAX_JOBS
if num_jobs is not None:
num_jobs = int(num_jobs)
logger.info("Using MAX_JOBS=%d as the number of jobs.", num_jobs)
else:
try:
# os.sched_getaffinity() isn't universally available, so fall
# back to os.cpu_count() if we get an error here.
num_jobs = len(os.sched_getaffinity(0))
except AttributeError:
num_jobs = os.cpu_count()
nvcc_threads = None
if _is_cuda() and CUDA_HOME is not None:
try:
nvcc_version = get_nvcc_cuda_version()
if nvcc_version >= Version("11.2"):
# `nvcc_threads` is either the value of the NVCC_THREADS
# environment variable (if defined) or 1.
# when it is set, we reduce `num_jobs` to avoid
# overloading the system.
nvcc_threads = envs.NVCC_THREADS
if nvcc_threads is not None:
nvcc_threads = int(nvcc_threads)
logger.info(
"Using NVCC_THREADS=%d as the number of nvcc threads.",
nvcc_threads,
)
else:
nvcc_threads = 1
num_jobs = max(1, num_jobs // nvcc_threads)
except Exception as e:
logger.warning("Failed to get NVCC version: %s", e)
return num_jobs, nvcc_threads
#
# Perform cmake configuration for a single extension.
#
def configure(self, ext: CMakeExtension) -> None:
# If we've already configured using the CMakeLists.txt for
# this extension, exit early.
if ext.cmake_lists_dir in cmake_build_ext.did_config:
return
cmake_build_ext.did_config[ext.cmake_lists_dir] = True
# Select the build type.
# Note: optimization level + debug info are set by the build type
default_cfg = "Debug" if self.debug else "RelWithDebInfo"
cfg = envs.CMAKE_BUILD_TYPE or default_cfg
cmake_args = [
f"-DCMAKE_BUILD_TYPE={cfg}",
f"-DVLLM_TARGET_DEVICE={VLLM_TARGET_DEVICE}",
]
verbose = envs.VERBOSE
if verbose:
cmake_args += ["-DCMAKE_VERBOSE_MAKEFILE=ON"]
if is_sccache_available():
cmake_args += [
"-DCMAKE_C_COMPILER_LAUNCHER=sccache",
"-DCMAKE_CXX_COMPILER_LAUNCHER=sccache",
"-DCMAKE_CUDA_COMPILER_LAUNCHER=sccache",
"-DCMAKE_HIP_COMPILER_LAUNCHER=sccache",
]
elif is_ccache_available():
cmake_args += [
"-DCMAKE_C_COMPILER_LAUNCHER=ccache",
"-DCMAKE_CXX_COMPILER_LAUNCHER=ccache",
"-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache",
"-DCMAKE_HIP_COMPILER_LAUNCHER=ccache",
]
# Pass the python executable to cmake so it can find an exact
# match.
cmake_args += [f"-DVLLM_PYTHON_EXECUTABLE={sys.executable}"]
# Pass the python path to cmake so it can reuse the build dependencies
# on subsequent calls to python.
cmake_args += ["-DVLLM_PYTHON_PATH={}".format(":".join(sys.path))]
# Override the base directory for FetchContent downloads to $ROOT/.deps
# This allows sharing dependencies between profiles,
# and plays more nicely with sccache.
# To override this, set the FETCHCONTENT_BASE_DIR environment variable.
fc_base_dir = os.path.join(ROOT_DIR, ".deps")
fc_base_dir = os.environ.get("FETCHCONTENT_BASE_DIR", fc_base_dir)
cmake_args += [f"-DFETCHCONTENT_BASE_DIR={fc_base_dir}"]
#
# Setup parallelism and build tool
#
num_jobs, nvcc_threads = self.compute_num_jobs()
if nvcc_threads:
cmake_args += [f"-DNVCC_THREADS={nvcc_threads}"]
if is_ninja_available():
build_tool = ["-G", "Ninja"]
cmake_args += [
"-DCMAKE_JOB_POOL_COMPILE:STRING=compile",
f"-DCMAKE_JOB_POOLS:STRING=compile={num_jobs}",
]
else:
# Default build tool to whatever cmake picks.
build_tool = []
# Make sure we use the nvcc from CUDA_HOME
if _is_cuda() and CUDA_HOME is not None:
cmake_args += [f"-DCMAKE_CUDA_COMPILER={CUDA_HOME}/bin/nvcc"]
other_cmake_args = os.environ.get("CMAKE_ARGS")
if other_cmake_args:
cmake_args += other_cmake_args.split()
subprocess.check_call(
["cmake", ext.cmake_lists_dir, *build_tool, *cmake_args],
cwd=self.build_temp,
)
def build_extensions(self) -> None:
# Ensure that CMake is present and working
try:
subprocess.check_output(["cmake", "--version"])
except OSError as e:
raise RuntimeError("Cannot find CMake executable") from e
# Create build directory if it does not exist.
if not os.path.exists(self.build_temp):
os.makedirs(self.build_temp)
targets = []
def target_name(s: str) -> str:
return s.removeprefix("vllm.").removeprefix("vllm_flash_attn.")
# Build all the extensions
for ext in self.extensions:
self.configure(ext)
targets.append(target_name(ext.name))
num_jobs, _ = self.compute_num_jobs()
build_args = [
"--build",
".",
f"-j={num_jobs}",
*[f"--target={name}" for name in targets],
]
subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
# Install the libraries
for ext in self.extensions:
# Install the extension into the proper location
outdir = Path(self.get_ext_fullpath(ext.name)).parent.absolute()
# Skip if the install directory is the same as the build directory
if outdir == self.build_temp:
continue
# CMake appends the extension prefix to the install path,
# and outdir already contains that prefix, so we need to remove it.
prefix = outdir
for _ in range(ext.name.count(".")):
prefix = prefix.parent
# prefix here should actually be the same for all components
install_args = [
"cmake",
"--install",
".",
"--prefix",
prefix,
"--component",
target_name(ext.name),
]
subprocess.check_call(install_args, cwd=self.build_temp)
def run(self):
# First, run the standard build_ext command to compile the extensions
super().run()
# copy vllm/vllm_flash_attn/**/*.py from self.build_lib to current
# directory so that they can be included in the editable build
import glob
files = glob.glob(
os.path.join(self.build_lib, "vllm", "vllm_flash_attn", "**", "*.py"),
recursive=True,
)
for file in files:
dst_file = os.path.join(
"vllm/vllm_flash_attn", file.split("vllm/vllm_flash_attn/")[-1]
)
print(f"Copying {file} to {dst_file}")
os.makedirs(os.path.dirname(dst_file), exist_ok=True)
self.copy_file(file, dst_file)
if _is_cuda():
# copy vllm/third_party/triton_kernels/**/*.py from self.build_lib
# to current directory so that they can be included in the editable
# build
print(
f"Copying {self.build_lib}/vllm/third_party/triton_kernels "
"to vllm/third_party/triton_kernels"
)
shutil.copytree(
f"{self.build_lib}/vllm/third_party/triton_kernels",
"vllm/third_party/triton_kernels",
dirs_exist_ok=True,
)
if _is_cuda():
# copy vendored deep_gemm package from build_lib to source tree
# for editable installs
deep_gemm_build = os.path.join(
self.build_lib, "vllm", "third_party", "deep_gemm"
)
if os.path.exists(deep_gemm_build):
print(f"Copying {deep_gemm_build} to vllm/third_party/deep_gemm")
shutil.copytree(
deep_gemm_build,
"vllm/third_party/deep_gemm",
dirs_exist_ok=True,
)
# copy vendored fmha_sm100 package from build_lib to source tree
# for editable installs
fmha_sm100_build = os.path.join(
self.build_lib, "vllm", "third_party", "fmha_sm100"
)
if os.path.exists(fmha_sm100_build):
print(f"Copying {fmha_sm100_build} to vllm/third_party/fmha_sm100")
shutil.copytree(
fmha_sm100_build,
"vllm/third_party/fmha_sm100",
dirs_exist_ok=True,
)
tml_fa4_build = os.path.join(
self.build_lib, "vllm", "third_party", "tml_fa4"
)
if os.path.exists(tml_fa4_build):
print(f"Copying {tml_fa4_build} to vllm/third_party/tml_fa4")
shutil.copytree(
tml_fa4_build,
"vllm/third_party/tml_fa4",
dirs_exist_ok=True,
)
class precompiled_build_ext(build_ext):
"""Disables extension building when using precompiled binaries."""
def run(self) -> None:
return
def build_extensions(self) -> None:
print("Skipping build_ext: using precompiled extensions.")
return
class precompiled_wheel_utils:
"""Extracts libraries and other files from an existing wheel."""
@staticmethod
def fetch_metadata_for_variant(
commit: str,
variant: str | None,
) -> tuple[list[dict], str]:
"""Fetches metadata for a specific variant of the precompiled wheel."""
import json
from urllib.request import urlopen
variant_dir = f"{variant}/" if variant is not None else ""
repo_url = f"https://wheels.vllm.ai/{commit}/{variant_dir}vllm/"
meta_url = repo_url + "metadata.json"
print(f"Trying to fetch nightly build metadata from {meta_url}")
with urlopen(meta_url) as resp:
wheels = json.loads(resp.read().decode("utf-8"))
return wheels, repo_url
@staticmethod
def detect_system_cuda_variant() -> str:
"""Auto-detect CUDA variant from torch, nvidia-smi, or env default."""
# Map CUDA major version to hosted wheel variants on wheels.vllm.ai
supported = {12: "cu129", 13: "cu130"}
# Respect explicitly set VLLM_MAIN_CUDA_VERSION
if envs.is_set("VLLM_MAIN_CUDA_VERSION"):
v = envs.VLLM_MAIN_CUDA_VERSION
print(f"Using VLLM_MAIN_CUDA_VERSION={v}")
return "cu" + v.replace(".", "")[:3]
# Try torch.version.cuda
cuda_version = None
try:
import torch
cuda_version = torch.version.cuda
except Exception:
pass
# Try nvidia-smi
if not cuda_version:
try:
out = subprocess.run(
["nvidia-smi"], capture_output=True, text=True, timeout=10
)
if m := re.search(r"CUDA Version:\s*(\d+\.\d+)", out.stdout):
cuda_version = m.group(1)
except Exception:
pass
# Fall back to default
if not cuda_version:
cuda_version = envs.VLLM_MAIN_CUDA_VERSION
# Map to supported variant
major = int(cuda_version.split(".")[0])
variant = supported.get(major, supported[max(supported)])
print(f"Detected CUDA {cuda_version}, using variant {variant}")
return variant
# Publication lags main by roughly 5-10 commits; anything much older is
# unlikely to still match the checkout's compiled sources.
WHEEL_SEARCH_DEPTH = 20
# Changes here make an older wheel's extensions unusable.
WHEEL_BLOCKING_PATHS = (
"csrc",
"cmake",
"CMakeLists.txt",
"pyproject.toml",
"vllm/_custom_ops.py",
)
# Changes here are usually harmless for an older wheel.
WHEEL_WARNING_PATHS = ("setup.py",)
@staticmethod
def find_published_wheel_commit(
base_commit: str, variant: str | None, arch: str
) -> str:
"""Find the nearest ancestor of base_commit that has a published wheel.
Raises:
ValueError: If no wheel is published within WHEEL_SEARCH_DEPTH
commits, or compiled sources changed since the nearest one.
"""
from urllib.error import HTTPError
utils = precompiled_wheel_utils
candidates = subprocess.check_output(
[
"git",
"rev-list",
"--first-parent",
f"--max-count={utils.WHEEL_SEARCH_DEPTH}",
base_commit,
],
text=True,
).splitlines()
for distance, commit in enumerate(candidates):
try:
wheels, _ = utils.fetch_metadata_for_variant(commit, variant)
except HTTPError as err:
if err.code != 404:
raise
continue
if not any(
wheel.get("package_name") == "vllm"
and arch in wheel.get("platform_tag", "")
for wheel in wheels
):
continue
if distance == 0:
return commit
diff = ["git", "diff", "--name-only", commit, base_commit, "--"]
blocking, warning = (
subprocess.check_output([*diff, *paths], text=True).strip()
for paths in (utils.WHEEL_BLOCKING_PATHS, utils.WHEEL_WARNING_PATHS)
)
if blocking:
raise ValueError(
f"No precompiled wheel is published yet for {base_commit}, and "
f"the nearest one ({commit}, {distance} commits older) was "
f"built from different compiled sources:\n{blocking}\n"
"Wait for wheel publication, rebase onto a commit with a "
"published wheel, build from source (unset "
"VLLM_USE_PRECOMPILED), or set VLLM_PRECOMPILED_WHEEL_COMMIT "
"to a full commit SHA to skip this check."
)
if warning:
logger.warning(
"Precompiled wheel %s predates changes to:\n%s\n"
"The packaged files may be stale.",
commit,
warning,
)
print(
f"No precompiled wheel is published yet for {base_commit}; using "
f"{commit} ({distance} commits older)"
)
return commit
raise ValueError(
f"No published precompiled wheel for variant {variant} and "
f"architecture {arch} within {utils.WHEEL_SEARCH_DEPTH} commits of "
f"{base_commit}. Wait for wheel publication or build vLLM from source."
)
@staticmethod
def determine_wheel_url() -> tuple[str, str | None]:
"""Try to determine the precompiled wheel URL or path to use.
The order of preference is:
1. user-specified wheel location (can be either local or remote, via
VLLM_PRECOMPILED_WHEEL_LOCATION)
2. user-specified variant (VLLM_PRECOMPILED_WHEEL_VARIANT) from nightly repo
or CUDA variant selected from VLLM_MAIN_CUDA_VERSION, torch, or nvidia-smi
If downloading from the nightly repo, the commit can be specified via
VLLM_PRECOMPILED_WHEEL_COMMIT; otherwise, local checkouts search backwards
from their merge-base with upstream main for a compatible published wheel.
"""
wheel_location = os.getenv("VLLM_PRECOMPILED_WHEEL_LOCATION", None)
if wheel_location is not None:
print(f"Using user-specified precompiled wheel location: {wheel_location}")
return wheel_location, None
else:
import platform
arch = platform.machine()
# try to fetch the wheel metadata from the nightly wheel repo,
# detecting CUDA variant from system if not specified
variant = os.getenv("VLLM_PRECOMPILED_WHEEL_VARIANT", None)
if variant is None:
variant = precompiled_wheel_utils.detect_system_cuda_variant()
commit = os.getenv("VLLM_PRECOMPILED_WHEEL_COMMIT", "").lower()
if not commit or len(commit) != 40:
print(
f"VLLM_PRECOMPILED_WHEEL_COMMIT not valid: {commit}"
", trying to fetch base commit in main branch"
)
commit = precompiled_wheel_utils.get_base_commit_in_main_branch()
if (
envs.VLLM_USE_PRECOMPILED
and not envs.VLLM_DOCKER_BUILD_CONTEXT
and commit != "nightly"
):
commit = precompiled_wheel_utils.find_published_wheel_commit(
commit, variant, arch
)
print(f"Using precompiled wheel commit {commit} with variant {variant}")
download_filename = None
try:
wheels, repo_url = precompiled_wheel_utils.fetch_metadata_for_variant(
commit, variant
)
except Exception as e:
raise RuntimeError(
"Failed to fetch precompiled wheel metadata for CUDA "
f"variant {variant!r} at commit {commit}. The "
"root/default variant is not used as a fallback because "
"its CUDA compatibility with the selected variant cannot "
"be verified. Provide a compatible wheel with "
"VLLM_PRECOMPILED_WHEEL_LOCATION or disable "
"VLLM_USE_PRECOMPILED to build from source."
) from e
# The metadata.json has the following format:
# see .buildkite/scripts/generate-nightly-index.py for details
"""[{
"package_name": "vllm",
"version": "0.11.2.dev278+gdbc3d9991",
"build_tag": null,
"python_tag": "cp38",
"abi_tag": "abi3",
"platform_tag": "manylinux1_x86_64",
"variant": null,
"filename": "vllm-0.11.2.dev278+gdbc3d9991-cp38-abi3-manylinux1_x86_64.whl",
"path": "../vllm-0.11.2.dev278%2Bgdbc3d9991-cp38-abi3-manylinux1_x86_64.whl"
},
...]"""
from urllib.parse import urljoin
for wheel in wheels:
# TODO: maybe check more compatibility later? (python_tag, abi_tag, etc)
if wheel.get("package_name") == "vllm" and arch in wheel.get(
"platform_tag", ""
):
print(f"Found precompiled wheel metadata: {wheel}")
if "path" not in wheel:
raise ValueError(f"Wheel metadata missing path: {wheel}")
wheel_url = urljoin(repo_url, wheel["path"])
download_filename = wheel.get("filename")
print(f"Using precompiled wheel URL: {wheel_url}")
break
else:
raise ValueError(
f"No precompiled vllm wheel found for architecture {arch} "
f"from repo {repo_url}. All available wheels: {wheels}"
)
return wheel_url, download_filename
@staticmethod
def extract_precompiled_and_patch_package(
wheel_url_or_path: str,
download_filename: str | None,
*,
extract_extensions: bool,
) -> dict:
import tempfile
import zipfile
temp_dir = None
try:
if not os.path.isfile(wheel_url_or_path):
# use provided filename first, then derive from URL
wheel_filename = download_filename or wheel_url_or_path.split("/")[-1]
temp_dir = tempfile.mkdtemp(prefix="vllm-wheels")
wheel_path = os.path.join(temp_dir, wheel_filename)
print(f"Downloading wheel from {wheel_url_or_path} to {wheel_path}")
from urllib.request import urlretrieve
urlretrieve(wheel_url_or_path, filename=wheel_path)
else:
wheel_path = wheel_url_or_path
print(f"Using existing wheel at {wheel_path}")
package_data_patch = {}
with zipfile.ZipFile(wheel_path) as wheel:
exact_members = set()
if extract_extensions:
exact_members.update(
{
"vllm/_C.abi3.so",
"vllm/_C_stable_libtorch.abi3.so",
"vllm/_moe_C_stable_libtorch.abi3.so",
"vllm/_qutlass_C.abi3.so",
"vllm/_flashmla_C.abi3.so",
"vllm/_flashmla_extension_C.abi3.so",
"vllm/_flashkda_C.abi3.so",
"vllm/_sparse_flashmla_C.abi3.so",
"vllm/_deepselect_C.abi3.so",
"vllm/vllm_flash_attn/_vllm_fa2_C.abi3.so",
"vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
"vllm/cumem_allocator.abi3.so",
"vllm/spinloop.abi3.so",
"vllm/fs_io_C.abi3.so",
}
)
flash_attn_regex = re.compile(
r"vllm/vllm_flash_attn/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py"
)
# __init__.py and flash_attn_interface.py are source-controlled
# in vllm and should not be overwritten (matches cmake exclusions)
flash_attn_files_to_skip = {
"vllm/vllm_flash_attn/__init__.py",
"vllm/vllm_flash_attn/flash_attn_interface.py",
}
triton_kernels_regex = re.compile(
r"vllm/third_party/triton_kernels/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py"
)
flashmla_regex = re.compile(
r"vllm/third_party/flashmla/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py"
)
# DeepGEMM: extract all files (.py, .so, .cuh, .h, .hpp, etc.)
deep_gemm_regex = re.compile(r"vllm/third_party/deep_gemm/.*")
fmha_sm100_regex = re.compile(r"vllm/third_party/fmha_sm100/.*")
tml_fa4_regex = re.compile(r"vllm/third_party/tml_fa4/.*")
file_members = []
for member in wheel.filelist:
if member.filename in exact_members:
file_members.append(member)
continue
if not extract_extensions:
continue
if (
(
flash_attn_regex.match(member.filename)
and member.filename not in flash_attn_files_to_skip
)
or triton_kernels_regex.match(member.filename)
or flashmla_regex.match(member.filename)
or deep_gemm_regex.match(member.filename)
or tml_fa4_regex.match(member.filename)
or fmha_sm100_regex.match(member.filename)
):
file_members.append(member)
for file in file_members:
print(f"[extract] {file.filename}")
target_path = os.path.join(".", file.filename)
os.makedirs(os.path.dirname(target_path), exist_ok=True)
with (
wheel.open(file.filename) as src,
open(target_path, "wb") as dst,
):
shutil.copyfileobj(src, dst)
mode = file.external_attr >> 16
if mode:
os.chmod(target_path, mode)
pkg = os.path.dirname(file.filename).replace("/", ".")
package_data_patch.setdefault(pkg, []).append(
os.path.basename(file.filename)
)
return package_data_patch
finally:
if temp_dir is not None:
print(f"Removing temporary directory {temp_dir}")
shutil.rmtree(temp_dir)
@staticmethod
def get_base_commit_in_main_branch() -> str:
try:
# Get the latest commit hash of the upstream main branch.
curl_cmd = [
"curl",
"-s",
"https://api.github.com/repos/vllm-project/vllm/commits/main",
]
github_token = os.getenv("GH_TOKEN", os.getenv("GITHUB_TOKEN"))
if github_token:
curl_cmd += [
"-H",
f"Authorization: token {github_token}",
]
resp_json = subprocess.check_output(curl_cmd).decode("utf-8")
upstream_main_commit = json.loads(resp_json)["sha"]
print(f"Upstream main branch latest commit: {upstream_main_commit}")
# In Docker build context, .git may be immutable or missing.
if envs.VLLM_DOCKER_BUILD_CONTEXT:
return upstream_main_commit
# Check if the upstream_main_commit exists in the local repo
try:
subprocess.check_output(
["git", "cat-file", "-e", f"{upstream_main_commit}"]
)
except subprocess.CalledProcessError:
# If not present, fetch it from the remote repository.
# Note that this does not update any local branches,
# but ensures that this commit ref and its history are
# available in our local repo.
subprocess.check_call(
["git", "fetch", "https://github.com/vllm-project/vllm", "main"]
)
base_commit = (
subprocess.check_output(
["git", "merge-base", upstream_main_commit, "HEAD"]
)
.decode("utf-8")
.strip()
)
return base_commit
except ValueError as err:
raise ValueError(err) from None
except Exception as err:
logger.warning(
"Failed to get the base commit in the main branch. "
"Using the nightly wheel. The libraries in this "
"wheel may not be compatible with your dev branch: %s",
err,
)
return "nightly"
def _no_device() -> bool:
return VLLM_TARGET_DEVICE == "empty"
def _is_cuda() -> bool:
has_cuda = torch.version.cuda is not None
return VLLM_TARGET_DEVICE == "cuda" and has_cuda and not _is_tpu()
def _is_tpu() -> bool:
return VLLM_TARGET_DEVICE == "tpu"
def _is_xpu() -> bool:
return VLLM_TARGET_DEVICE == "xpu"
def _build_custom_ops() -> bool:
return _is_cuda()
def get_nvcc_cuda_version() -> Version:
"""Get the CUDA version from nvcc.
Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
"""
assert CUDA_HOME is not None, "CUDA_HOME is not set"
nvcc_output = subprocess.check_output(
[CUDA_HOME + "/bin/nvcc", "-V"], universal_newlines=True
)
output = nvcc_output.split()
release_idx = output.index("release") + 1
nvcc_cuda_version = parse(output[release_idx].split(",")[0])
return nvcc_cuda_version
def get_vllm_version() -> str:
# Allow overriding the version. This is useful to build platform-specific
# wheels (e.g. CPU, TPU) without modifying the source.
if env_version := os.getenv("VLLM_VERSION_OVERRIDE"):
print(f"Overriding VLLM version with {env_version} from VLLM_VERSION_OVERRIDE")
os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = env_version
return get_version(
write_to="vllm/_version.py",
git_describe_command=VLLM_GIT_DESCRIBE_COMMAND,
)
version = get_version(
write_to="vllm/_version.py",
git_describe_command=VLLM_GIT_DESCRIBE_COMMAND,
)
sep = "+" if "+" not in version else "." # dev versions might contain +
if not envs.VLLM_SKIP_VERSION_SUFFIX:
if _no_device():
if envs.VLLM_TARGET_DEVICE == "empty":
version += f"{sep}empty"
elif _is_cuda():
if (
USE_PRECOMPILED_EXTENSIONS
and not envs.VLLM_SKIP_PRECOMPILED_VERSION_SUFFIX
):
version += f"{sep}precompiled"
else:
cuda_version = str(get_nvcc_cuda_version())
if cuda_version != envs.VLLM_MAIN_CUDA_VERSION:
cuda_version_str = cuda_version.replace(".", "")[:3]
# skip this for source tarball, required for pypi
if "sdist" not in sys.argv:
version += f"{sep}cu{cuda_version_str}"
elif _is_tpu():
version += f"{sep}tpu"
elif _is_xpu():
version += f"{sep}xpu"
else:
raise RuntimeError("Unknown runtime environment")
return version
def get_requirements() -> list[str]:
"""Get Python package dependencies from requirements.txt."""
requirements_dir = ROOT_DIR / "requirements"
def _read_requirements(filename: str) -> list[str]:
with open(requirements_dir / filename) as f:
requirements = f.read().strip().split("\n")
resolved_requirements = []
for line in requirements:
if line.startswith("-r "):
resolved_requirements += _read_requirements(line.split()[1])
elif (
not line.startswith("--")
and not line.startswith("#")
and line.strip() != ""
):
resolved_requirements.append(line)
return resolved_requirements
if _no_device():
requirements = _read_requirements("common.txt")
elif _is_cuda():
requirements = _read_requirements("cuda.txt")
cuda_major, cuda_minor = torch.version.cuda.split(".")
modified_requirements = []
for req in requirements:
if "vllm-flash-attn" in req and cuda_major != "12":
# vllm-flash-attn is built only for CUDA 12.x.
# Skip for other versions.
continue
if "flashinfer-cubin" in req or "flashinfer-jit-cache" in req:
# Not on PyPI (only https://flashinfer.ai/whl), so they
# cannot be wheel dependencies; flashinfer handles the
# absence of pre-compiled cubins/jit-cache at runtime.
continue
if "nvidia-cutlass-dsl[cu13]" in req and cuda_major == "12":
# [cu13] extra is the default; strip it on CUDA 12 builds.
req = req.replace("nvidia-cutlass-dsl[cu13]", "nvidia-cutlass-dsl")
if "humming-kernels[cu13]" in req and cuda_major == "12":
req = req.replace("humming-kernels[cu13]", "humming-kernels[cu12]")
modified_requirements.append(req)
requirements = modified_requirements
elif _is_tpu():
requirements = _read_requirements("tpu.txt")
elif _is_xpu():
requirements = _read_requirements("xpu.txt")
else:
raise ValueError("Unsupported platform, please use CUDA.")
return requirements
ext_modules = []
if _is_cuda():
ext_modules.append(CMakeExtension(name="vllm.cumem_allocator"))
# Optional since this doesn't get built (produce an .so file). This is just
# copying the relevant .py files from the source repository.
ext_modules.append(CMakeExtension(name="vllm.triton_kernels", optional=True))
if not _is_xpu():
ext_modules.append(CMakeExtension(name="vllm.spinloop"))
ext_modules.append(CMakeExtension(name="vllm.fs_io_C"))
if _is_cuda():
ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa2_C"))
if USE_PRECOMPILED_EXTENSIONS or (
CUDA_HOME and get_nvcc_cuda_version() >= Version("12.3")
):
# FA3 requires CUDA 12.3 or later
ext_modules.append(CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa3_C"))
# FA4 CuteDSL - Python-only component for FA4's cute DSL support
# Optional since this doesn't produce a .so file, just copies Python files
ext_modules.append(
CMakeExtension(name="vllm.vllm_flash_attn._vllm_fa4_cutedsl_C", optional=True)
)
if USE_PRECOMPILED_EXTENSIONS or (
CUDA_HOME and get_nvcc_cuda_version() >= Version("12.9")
):
# FlashMLA requires CUDA 12.9 or later
# Optional since this doesn't get built (produce an .so file) when
# not targeting a hopper system
ext_modules.append(CMakeExtension(name="vllm._flashmla_C", optional=True))
ext_modules.append(
CMakeExtension(name="vllm._flashmla_extension_C", optional=True)
)
# DeepSelect requires CUDA 12.9 or later (SM100a/SM103a only)
# Optional since it won't build on unsupported architectures
ext_modules.append(CMakeExtension(name="vllm._deepselect_C", optional=True))
if USE_PRECOMPILED_EXTENSIONS or (
CUDA_HOME and get_nvcc_cuda_version() >= Version("12.0")
):
ext_modules.append(CMakeExtension(name="vllm._flashkda_C", optional=True))
if envs.VLLM_USE_PRECOMPILED or (
CUDA_HOME and get_nvcc_cuda_version() >= Version("12.3")
):
# DeepGEMM requires CUDA 12.3+ (SM90/SM100)
# Optional since it won't build on unsupported architectures
ext_modules.append(CMakeExtension(name="vllm._deep_gemm_C", optional=True))
ext_modules.append(CMakeExtension(name="vllm._qutlass_C", optional=True))
# fmha_sm100 is a Python/CuTe-DSL package installed into vllm.third_party.
ext_modules.append(CMakeExtension(name="vllm.fmha_sm100", optional=True))
# tml-fa4 is copied into an isolated vllm.third_party package.
ext_modules.append(CMakeExtension(name="vllm.tml_fa4", optional=True))
if _build_custom_ops():
ext_modules.append(CMakeExtension(name="vllm._C_stable_libtorch"))
ext_modules.append(CMakeExtension(name="vllm._moe_C_stable_libtorch"))
package_data = {
"vllm": [
"py.typed",
"libs/*.so*",
"entrypoints/serve/instrumentator/static/*.js",
"entrypoints/serve/instrumentator/static/*.css",
"distributed/kv_transfer/kv_connector/v1/hf3fs/utils/*.cpp",
# Built-in multimodal chat template fallbacks (registry.py)
"transformers_utils/chat_templates/*.jinja",
"third_party/flash_linear_attention/LICENSE",
# DeepGEMM JIT include headers (vendored via cmake)
"third_party/deep_gemm/include/**/*.cuh",
"third_party/deep_gemm/include/**/*.h",
"third_party/deep_gemm/include/**/*.hpp",
# fmha_sm100 sparse CuTe-DSL helper kernels (vendored via cmake)
"third_party/fmha_sm100/csrc/**/*.cu",
"third_party/fmha_sm100/csrc/**/*.h",
"third_party/fmha_sm100/csrc/**/*.jinja",
"third_party/fmha_sm100/csrc/**/*.cu.jinja",
"third_party/fmha_sm100/cute/**/*.cu",
"third_party/fmha_sm100/cutlass/include/**/*.h",
"third_party/fmha_sm100/cutlass/include/**/*.hpp",
"third_party/fmha_sm100/cutlass/tools/util/include/**/*.h",
"third_party/fmha_sm100/cutlass/tools/util/include/**/*.hpp",
]
}
def add_vllm_package_data(filename: str) -> None:
vllm_files = package_data.setdefault("vllm", [])
if filename not in vllm_files:
vllm_files.append(filename)
# PEP 517 invokes setup.py for metadata before invoking the actual build.
if USE_PRECOMPILED_EXTENSIONS and not is_metadata_only_build():
wheel_url, download_filename = precompiled_wheel_utils.determine_wheel_url()
patch = precompiled_wheel_utils.extract_precompiled_and_patch_package(
wheel_url,
download_filename,
extract_extensions=USE_PRECOMPILED_EXTENSIONS,
)
for pkg, files in patch.items():
package_data.setdefault(pkg, []).extend(files)
if _no_device():
ext_modules = []
if not ext_modules:
cmdclass = {}
else:
cmdclass = {
"build_ext": precompiled_build_ext
if USE_PRECOMPILED_EXTENSIONS
else cmake_build_ext,
}
# Resolve the Python version first because get_vllm_version() may set
# SETUPTOOLS_SCM_PRETEND_VERSION.
vllm_version = get_vllm_version()
setup(
# static metadata should rather go in pyproject.toml
version=vllm_version,
ext_modules=ext_modules,
install_requires=get_requirements(),
extras_require={
"bench": ["pandas", "matplotlib", "seaborn", "datasets", "scipy", "plotly"],
"tensorizer": ["tensorizer==2.10.1"],
"fastsafetensors": ["fastsafetensors >= 0.3.3"],
"instanttensor": ["instanttensor >= 0.2.2"],
"runai": ["runai-model-streamer[s3,gcs,azure] >= 0.15.7"],
"audio": [
"av",
"scipy",
"soundfile",
"soxr",
"mistral_common[audio]",
], # Required for audio processing
"video": [], # Kept for backwards compatibility
# NVIDIA DeepStream (NVDEC) GPU video-decode backend. Linux x86-64
# only; also needs system GStreamer + libv4l (see docs).
"deepstream": ["nvidia-deepstream-videodecode-cu13>=9.0.2"],
"flashinfer": [], # Kept for backwards compatibility
"b12x": ["b12x==1.5.1"],
# Optional deps for Helion kernel development
# NOTE: When updating helion version, also update CI files:
# - .buildkite/test_areas/kernels.yaml
# - .buildkite/test-amd.yaml
"helion": ["helion==1.4.0"],
# Optional deps for gRPC server (vllm serve --grpc)
"grpc": ["smg-grpc-servicer[vllm] >= 0.5.2"],
# Optional deps for OpenTelemetry tracing
"otel": [
"opentelemetry-sdk>=1.26.0",
"opentelemetry-api>=1.26.0",
"opentelemetry-exporter-otlp>=1.26.0",
"opentelemetry-semantic-conventions-ai>=0.4.1",
],
# extra quantization plugin
"extra-quant": ["vllm-gguf-plugin>=0.0.2"],
},
cmdclass=cmdclass,
package_data=package_data,
)
Reference in New Issue View Git Blame Copy Permalink