[Build] Migrate vendored DeepGEMM from pybind to TORCH_LIBRARY (abi3) (#48962)
Signed-off-by: Chris Leonard <chleonar@redhat.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
co-authored by
Shengqi Chen
parent
cbe1f9740d
commit
cc73cca9c8
@@ -33,7 +33,8 @@ else()
|
||||
# port, the SM90 paged-MQA kv_block=32/next_n=4 port, configurable SwiGLU
|
||||
# alpha/beta, and SiTU for FP8/FP4 Mega MoE. Also includes the CUDA 12.x
|
||||
# layout header fix from vllm-project/DeepGEMM#12.
|
||||
set(_DEEPGEMM_UPSTREAM_TAG "e1f418c2a4f20818221f6b0e578b4c2f634d4c3f")
|
||||
# TORCH_LIBRARY build with a CPython 3.10 stable-ABI floor.
|
||||
set(_DEEPGEMM_UPSTREAM_TAG "1e1842a833699298f7afc02eefb2ed168fff6938")
|
||||
|
||||
set(_deepgemm_fc_root "${FETCHCONTENT_BASE_DIR}")
|
||||
if(NOT _deepgemm_fc_root)
|
||||
@@ -43,7 +44,7 @@ else()
|
||||
set(_deepgemm_bin "${_deepgemm_fc_root}/deepgemm-build")
|
||||
set(_deepgemm_sub "${_deepgemm_fc_root}/deepgemm-subbuild")
|
||||
|
||||
if(EXISTS "${_deepgemm_src}/csrc/python_api.cpp")
|
||||
if(EXISTS "${_deepgemm_src}/deep_gemm/_C.py")
|
||||
set(deepgemm_SOURCE_DIR "${_deepgemm_src}")
|
||||
set(deepgemm_BINARY_DIR "${_deepgemm_bin}")
|
||||
else()
|
||||
@@ -93,33 +94,17 @@ if(DEEPGEMM_ARCHS)
|
||||
#
|
||||
# DeepGEMM integration notes
|
||||
# --------------------------
|
||||
# We vendor DeepGEMM into vllm/third_party/deep_gemm/ and bundle a
|
||||
# `_C.cpython-X.Y-*.so` for every CPython in `requires-python`. The
|
||||
# per-Python build is delegated to tools/build_deepgemm_C.py.
|
||||
# We vendor DeepGEMM into vllm/third_party/deep_gemm/ and bundle:
|
||||
# - deep_gemm/_C.py (Python shim over torch.ops.deep_gemm)
|
||||
# - deep_gemm/_C_extension.abi3.so (single limited-API extension)
|
||||
# The build is delegated to tools/build_deepgemm_C.py (setup.py build_ext).
|
||||
#
|
||||
# Why per-Python: DeepGEMM's binding uses PYBIND11_MODULE, which links
|
||||
# private CPython symbols — a single `_C.abi3.so` is not viable today
|
||||
# (see #41476 / #41512 for the failed attempt).
|
||||
#
|
||||
# TODOs (tracked in vllm-project/vllm#42431):
|
||||
# - Replace DeepGEMM's pybind11 binding with a TORCH_LIBRARY + shim
|
||||
# binding (cf. vllm-flash-attention/csrc/common/pytorch_shim.h) to
|
||||
# collapse to one `_C.abi3.so`. Needs either an upstream change or
|
||||
# a maintained binding fork in vLLM.
|
||||
# - AOT-compile DeepGEMM's CUDA kernels instead of runtime JIT to drop
|
||||
# the vendored CUTLASS/CCCL headers and the CUDA-toolkit-at-runtime
|
||||
# requirement.
|
||||
# TODO: AOT-compile DeepGEMM's CUDA kernels instead of runtime JIT to drop
|
||||
# the vendored CUTLASS/CCCL headers and the CUDA-toolkit-at-runtime
|
||||
# requirement.
|
||||
#
|
||||
|
||||
# DEEPGEMM_PYTHON_INTERPRETERS: ":"-separated target Python paths.
|
||||
# Empty/unset → fall back to the build interpreter (editable installs).
|
||||
# (Empty-but-set env vars test as DEFINED in cmake — treat as unset.)
|
||||
if(NOT "$ENV{DEEPGEMM_PYTHON_INTERPRETERS}" STREQUAL "")
|
||||
string(REPLACE ":" ";" _dg_pythons "$ENV{DEEPGEMM_PYTHON_INTERPRETERS}")
|
||||
else()
|
||||
set(_dg_pythons "${Python_EXECUTABLE}")
|
||||
endif()
|
||||
message(STATUS "DeepGEMM _C will be built for: ${_dg_pythons}")
|
||||
message(STATUS "DeepGEMM extension will be built with: ${Python_EXECUTABLE}")
|
||||
|
||||
# add_custom_command does no implicit header scanning; glob explicitly so
|
||||
# header-only edits in DeepGEMM/cutlass/deep_jit re-trigger the rebuild.
|
||||
@@ -132,40 +117,29 @@ if(DEEPGEMM_ARCHS)
|
||||
"${deepgemm_SOURCE_DIR}/third-party/deep_jit/include/*.h"
|
||||
"${deepgemm_SOURCE_DIR}/third-party/deep_jit/include/*.hpp")
|
||||
|
||||
set(_dg_markers)
|
||||
set(_dg_seen_soabis)
|
||||
foreach(_pybin IN LISTS _dg_pythons)
|
||||
execute_process(
|
||||
COMMAND "${_pybin}" -c
|
||||
"import sysconfig; print(sysconfig.get_config_var('SOABI'))"
|
||||
OUTPUT_VARIABLE _dg_soabi
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
COMMAND_ERROR_IS_FATAL ANY)
|
||||
# Dedup interpreters that resolve to the same CPython.
|
||||
if(_dg_soabi IN_LIST _dg_seen_soabis)
|
||||
continue()
|
||||
endif()
|
||||
list(APPEND _dg_seen_soabis "${_dg_soabi}")
|
||||
set(_dg_dir "${CMAKE_CURRENT_BINARY_DIR}/deepgemm_C_${_dg_soabi}")
|
||||
set(_dg_marker "${_dg_dir}/.built")
|
||||
add_custom_command(
|
||||
OUTPUT "${_dg_marker}"
|
||||
COMMAND "${Python_EXECUTABLE}"
|
||||
"${CMAKE_SOURCE_DIR}/tools/build_deepgemm_C.py"
|
||||
"${deepgemm_SOURCE_DIR}" "${_dg_dir}" "${_pybin}"
|
||||
COMMAND "${CMAKE_COMMAND}" -E touch "${_dg_marker}"
|
||||
DEPENDS "${CMAKE_SOURCE_DIR}/tools/build_deepgemm_C.py"
|
||||
"${deepgemm_SOURCE_DIR}/csrc/python_api.cpp"
|
||||
${_dg_headers}
|
||||
COMMENT "Building DeepGEMM _C for ${_pybin}"
|
||||
VERBATIM)
|
||||
list(APPEND _dg_markers "${_dg_marker}")
|
||||
install(DIRECTORY "${_dg_dir}/"
|
||||
DESTINATION vllm/third_party/deep_gemm
|
||||
COMPONENT _deep_gemm_C
|
||||
FILES_MATCHING PATTERN "_C.cpython-*.so")
|
||||
endforeach()
|
||||
add_custom_target(_deep_gemm_C ALL DEPENDS ${_dg_markers})
|
||||
set(_dg_dir "${CMAKE_CURRENT_BINARY_DIR}/deepgemm_C")
|
||||
set(_dg_marker "${_dg_dir}/.built")
|
||||
add_custom_command(
|
||||
OUTPUT "${_dg_marker}"
|
||||
COMMAND "${Python_EXECUTABLE}"
|
||||
"${CMAKE_SOURCE_DIR}/tools/build_deepgemm_C.py"
|
||||
"${deepgemm_SOURCE_DIR}" "${_dg_dir}"
|
||||
COMMAND "${CMAKE_COMMAND}" -E touch "${_dg_marker}"
|
||||
DEPENDS "${CMAKE_SOURCE_DIR}/tools/build_deepgemm_C.py"
|
||||
"${deepgemm_SOURCE_DIR}/csrc/python_api.cpp"
|
||||
"${deepgemm_SOURCE_DIR}/deep_gemm/_C.py"
|
||||
"${deepgemm_SOURCE_DIR}/setup.py"
|
||||
${_dg_headers}
|
||||
COMMENT "Building DeepGEMM _C_extension (abi3)"
|
||||
VERBATIM)
|
||||
add_custom_target(_deep_gemm_C ALL DEPENDS "${_dg_marker}")
|
||||
|
||||
install(DIRECTORY "${_dg_dir}/"
|
||||
DESTINATION vllm/third_party/deep_gemm
|
||||
COMPONENT _deep_gemm_C
|
||||
FILES_MATCHING
|
||||
PATTERN "_C.py"
|
||||
PATTERN "_C_extension*.so")
|
||||
|
||||
#
|
||||
# Vendor DeepGEMM Python package files
|
||||
|
||||
+34
-66
@@ -1,85 +1,53 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Build DeepGEMM's `_C` pybind11 extension for <TARGET_PY>.
|
||||
"""Build DeepGEMM's TORCH_LIBRARY extension and copy vendored artifacts.
|
||||
|
||||
Driven from cmake/external_projects/deepgemm.cmake. The driver runs against
|
||||
the build interpreter's torch; <TARGET_PY> is only consulted for INCLUDEPY
|
||||
and SOABI, so target venvs don't need torch installed.
|
||||
DeepGEMM now registers ops via TORCH_LIBRARY into ``deep_gemm._C_extension``
|
||||
(abi3) and exposes the legacy API through ``deep_gemm/_C.py``. This driver
|
||||
delegates to DeepGEMM's ``setup.py build_ext --inplace`` and copies the shim
|
||||
plus extension into the cmake output directory.
|
||||
|
||||
Usage: python build_deepgemm_C.py <DEEPGEMM_SRC_DIR> <OUTPUT_DIR> <TARGET_PY>
|
||||
Usage: python build_deepgemm_C.py <DEEPGEMM_SRC_DIR> <OUTPUT_DIR>
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from torch.utils import cpp_extension
|
||||
|
||||
if len(sys.argv) != 4:
|
||||
sys.exit(f"usage: {sys.argv[0]} <SRC> <OUT> <TARGET_PY>")
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit(f"usage: {sys.argv[0]} <SRC> <OUT>")
|
||||
|
||||
src = Path(sys.argv[1]).resolve()
|
||||
out = Path(sys.argv[2]).resolve()
|
||||
target_py = sys.argv[3]
|
||||
_pkg = src / "deep_gemm"
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
info = json.loads(
|
||||
subprocess.check_output(
|
||||
[
|
||||
target_py,
|
||||
"-c",
|
||||
"import sysconfig, json; "
|
||||
"print(json.dumps({k: sysconfig.get_config_var(k) "
|
||||
"for k in ('EXT_SUFFIX', 'INCLUDEPY')}))",
|
||||
]
|
||||
).decode()
|
||||
if not (_pkg / "_C.py").is_file():
|
||||
sys.exit(
|
||||
f"DeepGEMM source at {src} is missing deep_gemm/_C.py; "
|
||||
"expected TORCH_LIBRARY migration layout"
|
||||
)
|
||||
|
||||
# Avoid DeepGEMM's clean-git assertion when vendoring a local dirty tree.
|
||||
env = os.environ.copy()
|
||||
env.pop("DG_SKIP_CUDA_BUILD", None)
|
||||
|
||||
print(f"[build_deepgemm_C] building in {src} with {sys.executable}", flush=True)
|
||||
subprocess.check_call(
|
||||
[sys.executable, "setup.py", "build_ext", "--inplace"],
|
||||
cwd=src,
|
||||
env=env,
|
||||
)
|
||||
|
||||
cuda_home = cpp_extension.CUDA_HOME
|
||||
if cuda_home is None:
|
||||
sys.exit("CUDA_HOME not found; cannot build DeepGEMM _C")
|
||||
# CCCL lives outside the standard CUDAToolkit search (mirrors DeepGEMM's setup.py).
|
||||
includes = [
|
||||
info["INCLUDEPY"],
|
||||
f"{cuda_home}/include",
|
||||
f"{cuda_home}/include/cccl",
|
||||
str(src / "csrc"),
|
||||
str(src / "deep_gemm/include"),
|
||||
str(src / "third-party/cutlass/include"),
|
||||
str(src / "third-party/cutlass/tools/util/include"),
|
||||
str(src / "third-party/deep_jit/include"),
|
||||
*cpp_extension.include_paths(device_type="cuda"),
|
||||
]
|
||||
shim = _pkg / "_C.py"
|
||||
shutil.copy2(shim, out / shim.name)
|
||||
|
||||
cmd = [
|
||||
os.environ.get("CXX", "g++"),
|
||||
"-shared",
|
||||
"-fPIC",
|
||||
"-std=c++20",
|
||||
"-O3",
|
||||
"-g0",
|
||||
"-Wno-psabi",
|
||||
"-Wno-deprecated-declarations",
|
||||
"-DTORCH_API_INCLUDE_EXTENSION_H",
|
||||
"-DTORCH_EXTENSION_NAME=_C",
|
||||
f"-D_GLIBCXX_USE_CXX11_ABI={int(torch.compiled_with_cxx11_abi())}",
|
||||
*(f"-I{p}" for p in includes),
|
||||
str(src / "csrc/python_api.cpp"),
|
||||
*(f"-L{p}" for p in cpp_extension.library_paths(device_type="cuda")),
|
||||
f"-L{cuda_home}/lib64",
|
||||
"-ltorch",
|
||||
"-ltorch_python",
|
||||
"-ltorch_cpu",
|
||||
"-ltorch_cuda",
|
||||
"-lc10",
|
||||
"-lc10_cuda",
|
||||
"-lcudart",
|
||||
"-lnvrtc",
|
||||
"-o",
|
||||
str(out / f"_C{info['EXT_SUFFIX']}"),
|
||||
]
|
||||
print("[build_deepgemm_C] " + " ".join(cmd), flush=True)
|
||||
subprocess.check_call(cmd)
|
||||
so = _pkg / "_C_extension.abi3.so"
|
||||
if not so.is_file():
|
||||
sys.exit(f"DeepGEMM build did not produce {so}")
|
||||
for stale in out.glob("_C_extension*.so"):
|
||||
stale.unlink()
|
||||
shutil.copy2(so, out / so.name)
|
||||
print(f"[build_deepgemm_C] installed {so.name} -> {out}", flush=True)
|
||||
|
||||
@@ -1,41 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Assert the installed vLLM has a `_C.cpython-X.Y-*.so` for every CPython
|
||||
covered by `requires-python`. Fails closed if a Python's `.so` is missing
|
||||
from the wheel — i.e. the regression that surfaced in #41476/#41512.
|
||||
"""Assert the vendored DeepGEMM package has the TORCH_LIBRARY binding layout.
|
||||
|
||||
Run from a CI test job after vLLM is installed, e.g. the H100 deepgemm
|
||||
kernel tests in .buildkite/test_areas/kernels.yaml.
|
||||
Expects ``deep_gemm/_C.py`` plus a single ``_C_extension*.so`` (abi3) under
|
||||
``vllm.third_party.deep_gemm``. Run after vLLM is installed, e.g. the H100
|
||||
deepgemm kernel tests in .buildkite/test_areas/kernels.yaml.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import regex as re
|
||||
import tomllib
|
||||
|
||||
SO_RE = re.compile(r"^_C\.cpython-(\d)(\d+)-")
|
||||
def main() -> int:
|
||||
spec = importlib.util.find_spec("vllm.third_party.deep_gemm")
|
||||
if spec is None or spec.origin is None:
|
||||
print(
|
||||
"vllm.third_party.deep_gemm not importable; is vllm installed?",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
pkg_dir = Path(spec.origin).parent
|
||||
|
||||
shim = pkg_dir / "_C.py"
|
||||
so_files = sorted(pkg_dir.glob("_C_extension*.so"))
|
||||
missing = []
|
||||
if not shim.is_file():
|
||||
missing.append("_C.py")
|
||||
if len(so_files) != 1:
|
||||
missing.append("exactly one _C_extension*.so")
|
||||
|
||||
print(
|
||||
f"deepgemm vendored binding: shim={shim.is_file()}, "
|
||||
f"extensions={[p.name for p in so_files]}"
|
||||
)
|
||||
if missing:
|
||||
print(f"missing: {missing}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def required_pythons() -> list[str]:
|
||||
pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml"
|
||||
spec = tomllib.loads(pyproject.read_text())["project"]["requires-python"]
|
||||
m = re.match(r">=3\.(\d+),<3\.(\d+)", spec)
|
||||
if not m:
|
||||
sys.exit(f"unexpected requires-python format: {spec!r}")
|
||||
return [f"3.{v}" for v in range(int(m[1]), int(m[2]))]
|
||||
|
||||
|
||||
spec = importlib.util.find_spec("vllm.third_party.deep_gemm")
|
||||
if spec is None or spec.origin is None:
|
||||
sys.exit("vllm.third_party.deep_gemm not importable; is vllm installed?")
|
||||
pkg_dir = Path(spec.origin).parent
|
||||
|
||||
found = {f"{m[1]}.{m[2]}" for f in os.listdir(pkg_dir) if (m := SO_RE.match(f))}
|
||||
required = required_pythons()
|
||||
missing = [v for v in required if v not in found]
|
||||
print(f"deepgemm _C: found {sorted(found)}, required {required}, missing {missing}")
|
||||
sys.exit(1 if missing else 0)
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -10,7 +10,7 @@ DEEPGEMM_GIT_REPO="https://github.com/vllm-project/DeepGEMM.git"
|
||||
# NOTE: This targets the vLLM fork's dev branch, which carries the sm120
|
||||
# and sm90 paged-MQA ports plus the SwiGLU alpha/beta and SiTU Mega MoE
|
||||
# activations, plus the CUDA 12.x layout header fix from vllm-project/DeepGEMM#12.
|
||||
DEEPGEMM_GIT_REF="e1f418c2a4f20818221f6b0e578b4c2f634d4c3f"
|
||||
DEEPGEMM_GIT_REF="1e1842a833699298f7afc02eefb2ed168fff6938"
|
||||
WHEEL_DIR=""
|
||||
|
||||
# Parse command line arguments
|
||||
@@ -93,6 +93,8 @@ pushd "$INSTALL_DIR/deepgemm"
|
||||
|
||||
# Checkout the specific reference
|
||||
git checkout "$DEEPGEMM_GIT_REF"
|
||||
git submodule sync --recursive
|
||||
git submodule update --init --recursive
|
||||
|
||||
# Clean previous build artifacts
|
||||
# (Based on https://github.com/deepseek-ai/DeepGEMM/blob/main/install.sh)
|
||||
|
||||
@@ -179,6 +179,7 @@ def _deepgemm_fp8_gemm_nt_warmup(
|
||||
|
||||
device = w.device
|
||||
a1q = torch.empty((max_tokens, k), device=device, dtype=torch.float8_e4m3fn)
|
||||
# Must be initialized (UE8M0 packing asserts zero sign/mantissa bits).
|
||||
a1q_scales = torch.zeros(
|
||||
(max_tokens, k // block_m), device=device, dtype=torch.float32
|
||||
)
|
||||
@@ -284,6 +285,7 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
((MAX_M, k // block_m), torch.float32),
|
||||
((MAX_M, n), torch.bfloat16),
|
||||
)
|
||||
# Must be initialized (UE8M0 packing asserts zero sign/mantissa bits).
|
||||
a1q_scales.zero_()
|
||||
|
||||
for num_tokens, align_used, expert_ids in warmup_cases:
|
||||
|
||||
Reference in New Issue
Block a user