diff --git a/cmake/external_projects/deepgemm.cmake b/cmake/external_projects/deepgemm.cmake index a17031cc5e..1dcee345f8 100644 --- a/cmake/external_projects/deepgemm.cmake +++ b/cmake/external_projects/deepgemm.cmake @@ -33,7 +33,8 @@ else() # port, the SM90 paged-MQA kv_block=32/next_n=4 port, configurable SwiGLU # alpha/beta, and SiTU for FP8/FP4 Mega MoE. Also includes the CUDA 12.x # layout header fix from vllm-project/DeepGEMM#12. - set(_DEEPGEMM_UPSTREAM_TAG "e1f418c2a4f20818221f6b0e578b4c2f634d4c3f") + # TORCH_LIBRARY build with a CPython 3.10 stable-ABI floor. + set(_DEEPGEMM_UPSTREAM_TAG "1e1842a833699298f7afc02eefb2ed168fff6938") set(_deepgemm_fc_root "${FETCHCONTENT_BASE_DIR}") if(NOT _deepgemm_fc_root) @@ -43,7 +44,7 @@ else() set(_deepgemm_bin "${_deepgemm_fc_root}/deepgemm-build") set(_deepgemm_sub "${_deepgemm_fc_root}/deepgemm-subbuild") - if(EXISTS "${_deepgemm_src}/csrc/python_api.cpp") + if(EXISTS "${_deepgemm_src}/deep_gemm/_C.py") set(deepgemm_SOURCE_DIR "${_deepgemm_src}") set(deepgemm_BINARY_DIR "${_deepgemm_bin}") else() @@ -93,33 +94,17 @@ if(DEEPGEMM_ARCHS) # # DeepGEMM integration notes # -------------------------- - # We vendor DeepGEMM into vllm/third_party/deep_gemm/ and bundle a - # `_C.cpython-X.Y-*.so` for every CPython in `requires-python`. The - # per-Python build is delegated to tools/build_deepgemm_C.py. + # We vendor DeepGEMM into vllm/third_party/deep_gemm/ and bundle: + # - deep_gemm/_C.py (Python shim over torch.ops.deep_gemm) + # - deep_gemm/_C_extension.abi3.so (single limited-API extension) + # The build is delegated to tools/build_deepgemm_C.py (setup.py build_ext). # - # Why per-Python: DeepGEMM's binding uses PYBIND11_MODULE, which links - # private CPython symbols — a single `_C.abi3.so` is not viable today - # (see #41476 / #41512 for the failed attempt). - # - # TODOs (tracked in vllm-project/vllm#42431): - # - Replace DeepGEMM's pybind11 binding with a TORCH_LIBRARY + shim - # binding (cf. vllm-flash-attention/csrc/common/pytorch_shim.h) to - # collapse to one `_C.abi3.so`. Needs either an upstream change or - # a maintained binding fork in vLLM. - # - AOT-compile DeepGEMM's CUDA kernels instead of runtime JIT to drop - # the vendored CUTLASS/CCCL headers and the CUDA-toolkit-at-runtime - # requirement. + # TODO: AOT-compile DeepGEMM's CUDA kernels instead of runtime JIT to drop + # the vendored CUTLASS/CCCL headers and the CUDA-toolkit-at-runtime + # requirement. # - # DEEPGEMM_PYTHON_INTERPRETERS: ":"-separated target Python paths. - # Empty/unset → fall back to the build interpreter (editable installs). - # (Empty-but-set env vars test as DEFINED in cmake — treat as unset.) - if(NOT "$ENV{DEEPGEMM_PYTHON_INTERPRETERS}" STREQUAL "") - string(REPLACE ":" ";" _dg_pythons "$ENV{DEEPGEMM_PYTHON_INTERPRETERS}") - else() - set(_dg_pythons "${Python_EXECUTABLE}") - endif() - message(STATUS "DeepGEMM _C will be built for: ${_dg_pythons}") + message(STATUS "DeepGEMM extension will be built with: ${Python_EXECUTABLE}") # add_custom_command does no implicit header scanning; glob explicitly so # header-only edits in DeepGEMM/cutlass/deep_jit re-trigger the rebuild. @@ -132,40 +117,29 @@ if(DEEPGEMM_ARCHS) "${deepgemm_SOURCE_DIR}/third-party/deep_jit/include/*.h" "${deepgemm_SOURCE_DIR}/third-party/deep_jit/include/*.hpp") - set(_dg_markers) - set(_dg_seen_soabis) - foreach(_pybin IN LISTS _dg_pythons) - execute_process( - COMMAND "${_pybin}" -c - "import sysconfig; print(sysconfig.get_config_var('SOABI'))" - OUTPUT_VARIABLE _dg_soabi - OUTPUT_STRIP_TRAILING_WHITESPACE - COMMAND_ERROR_IS_FATAL ANY) - # Dedup interpreters that resolve to the same CPython. - if(_dg_soabi IN_LIST _dg_seen_soabis) - continue() - endif() - list(APPEND _dg_seen_soabis "${_dg_soabi}") - set(_dg_dir "${CMAKE_CURRENT_BINARY_DIR}/deepgemm_C_${_dg_soabi}") - set(_dg_marker "${_dg_dir}/.built") - add_custom_command( - OUTPUT "${_dg_marker}" - COMMAND "${Python_EXECUTABLE}" - "${CMAKE_SOURCE_DIR}/tools/build_deepgemm_C.py" - "${deepgemm_SOURCE_DIR}" "${_dg_dir}" "${_pybin}" - COMMAND "${CMAKE_COMMAND}" -E touch "${_dg_marker}" - DEPENDS "${CMAKE_SOURCE_DIR}/tools/build_deepgemm_C.py" - "${deepgemm_SOURCE_DIR}/csrc/python_api.cpp" - ${_dg_headers} - COMMENT "Building DeepGEMM _C for ${_pybin}" - VERBATIM) - list(APPEND _dg_markers "${_dg_marker}") - install(DIRECTORY "${_dg_dir}/" - DESTINATION vllm/third_party/deep_gemm - COMPONENT _deep_gemm_C - FILES_MATCHING PATTERN "_C.cpython-*.so") - endforeach() - add_custom_target(_deep_gemm_C ALL DEPENDS ${_dg_markers}) + set(_dg_dir "${CMAKE_CURRENT_BINARY_DIR}/deepgemm_C") + set(_dg_marker "${_dg_dir}/.built") + add_custom_command( + OUTPUT "${_dg_marker}" + COMMAND "${Python_EXECUTABLE}" + "${CMAKE_SOURCE_DIR}/tools/build_deepgemm_C.py" + "${deepgemm_SOURCE_DIR}" "${_dg_dir}" + COMMAND "${CMAKE_COMMAND}" -E touch "${_dg_marker}" + DEPENDS "${CMAKE_SOURCE_DIR}/tools/build_deepgemm_C.py" + "${deepgemm_SOURCE_DIR}/csrc/python_api.cpp" + "${deepgemm_SOURCE_DIR}/deep_gemm/_C.py" + "${deepgemm_SOURCE_DIR}/setup.py" + ${_dg_headers} + COMMENT "Building DeepGEMM _C_extension (abi3)" + VERBATIM) + add_custom_target(_deep_gemm_C ALL DEPENDS "${_dg_marker}") + + install(DIRECTORY "${_dg_dir}/" + DESTINATION vllm/third_party/deep_gemm + COMPONENT _deep_gemm_C + FILES_MATCHING + PATTERN "_C.py" + PATTERN "_C_extension*.so") # # Vendor DeepGEMM Python package files diff --git a/tools/build_deepgemm_C.py b/tools/build_deepgemm_C.py index f4438c6755..36824318c0 100644 --- a/tools/build_deepgemm_C.py +++ b/tools/build_deepgemm_C.py @@ -1,85 +1,53 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Build DeepGEMM's `_C` pybind11 extension for . +"""Build DeepGEMM's TORCH_LIBRARY extension and copy vendored artifacts. -Driven from cmake/external_projects/deepgemm.cmake. The driver runs against -the build interpreter's torch; is only consulted for INCLUDEPY -and SOABI, so target venvs don't need torch installed. +DeepGEMM now registers ops via TORCH_LIBRARY into ``deep_gemm._C_extension`` +(abi3) and exposes the legacy API through ``deep_gemm/_C.py``. This driver +delegates to DeepGEMM's ``setup.py build_ext --inplace`` and copies the shim +plus extension into the cmake output directory. -Usage: python build_deepgemm_C.py +Usage: python build_deepgemm_C.py """ -import json import os +import shutil import subprocess import sys from pathlib import Path -import torch -from torch.utils import cpp_extension - -if len(sys.argv) != 4: - sys.exit(f"usage: {sys.argv[0]} ") +if len(sys.argv) != 3: + sys.exit(f"usage: {sys.argv[0]} ") src = Path(sys.argv[1]).resolve() out = Path(sys.argv[2]).resolve() -target_py = sys.argv[3] +_pkg = src / "deep_gemm" out.mkdir(parents=True, exist_ok=True) -info = json.loads( - subprocess.check_output( - [ - target_py, - "-c", - "import sysconfig, json; " - "print(json.dumps({k: sysconfig.get_config_var(k) " - "for k in ('EXT_SUFFIX', 'INCLUDEPY')}))", - ] - ).decode() +if not (_pkg / "_C.py").is_file(): + sys.exit( + f"DeepGEMM source at {src} is missing deep_gemm/_C.py; " + "expected TORCH_LIBRARY migration layout" + ) + +# Avoid DeepGEMM's clean-git assertion when vendoring a local dirty tree. +env = os.environ.copy() +env.pop("DG_SKIP_CUDA_BUILD", None) + +print(f"[build_deepgemm_C] building in {src} with {sys.executable}", flush=True) +subprocess.check_call( + [sys.executable, "setup.py", "build_ext", "--inplace"], + cwd=src, + env=env, ) -cuda_home = cpp_extension.CUDA_HOME -if cuda_home is None: - sys.exit("CUDA_HOME not found; cannot build DeepGEMM _C") -# CCCL lives outside the standard CUDAToolkit search (mirrors DeepGEMM's setup.py). -includes = [ - info["INCLUDEPY"], - f"{cuda_home}/include", - f"{cuda_home}/include/cccl", - str(src / "csrc"), - str(src / "deep_gemm/include"), - str(src / "third-party/cutlass/include"), - str(src / "third-party/cutlass/tools/util/include"), - str(src / "third-party/deep_jit/include"), - *cpp_extension.include_paths(device_type="cuda"), -] +shim = _pkg / "_C.py" +shutil.copy2(shim, out / shim.name) -cmd = [ - os.environ.get("CXX", "g++"), - "-shared", - "-fPIC", - "-std=c++20", - "-O3", - "-g0", - "-Wno-psabi", - "-Wno-deprecated-declarations", - "-DTORCH_API_INCLUDE_EXTENSION_H", - "-DTORCH_EXTENSION_NAME=_C", - f"-D_GLIBCXX_USE_CXX11_ABI={int(torch.compiled_with_cxx11_abi())}", - *(f"-I{p}" for p in includes), - str(src / "csrc/python_api.cpp"), - *(f"-L{p}" for p in cpp_extension.library_paths(device_type="cuda")), - f"-L{cuda_home}/lib64", - "-ltorch", - "-ltorch_python", - "-ltorch_cpu", - "-ltorch_cuda", - "-lc10", - "-lc10_cuda", - "-lcudart", - "-lnvrtc", - "-o", - str(out / f"_C{info['EXT_SUFFIX']}"), -] -print("[build_deepgemm_C] " + " ".join(cmd), flush=True) -subprocess.check_call(cmd) +so = _pkg / "_C_extension.abi3.so" +if not so.is_file(): + sys.exit(f"DeepGEMM build did not produce {so}") +for stale in out.glob("_C_extension*.so"): + stale.unlink() +shutil.copy2(so, out / so.name) +print(f"[build_deepgemm_C] installed {so.name} -> {out}", flush=True) diff --git a/tools/check_wheel_deepgemm.py b/tools/check_wheel_deepgemm.py index 6f8a03ffd3..fcc35ead58 100644 --- a/tools/check_wheel_deepgemm.py +++ b/tools/check_wheel_deepgemm.py @@ -1,41 +1,45 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Assert the installed vLLM has a `_C.cpython-X.Y-*.so` for every CPython -covered by `requires-python`. Fails closed if a Python's `.so` is missing -from the wheel — i.e. the regression that surfaced in #41476/#41512. +"""Assert the vendored DeepGEMM package has the TORCH_LIBRARY binding layout. -Run from a CI test job after vLLM is installed, e.g. the H100 deepgemm -kernel tests in .buildkite/test_areas/kernels.yaml. +Expects ``deep_gemm/_C.py`` plus a single ``_C_extension*.so`` (abi3) under +``vllm.third_party.deep_gemm``. Run after vLLM is installed, e.g. the H100 +deepgemm kernel tests in .buildkite/test_areas/kernels.yaml. """ import importlib.util -import os import sys from pathlib import Path -import regex as re -import tomllib -SO_RE = re.compile(r"^_C\.cpython-(\d)(\d+)-") +def main() -> int: + spec = importlib.util.find_spec("vllm.third_party.deep_gemm") + if spec is None or spec.origin is None: + print( + "vllm.third_party.deep_gemm not importable; is vllm installed?", + file=sys.stderr, + ) + return 1 + pkg_dir = Path(spec.origin).parent + + shim = pkg_dir / "_C.py" + so_files = sorted(pkg_dir.glob("_C_extension*.so")) + missing = [] + if not shim.is_file(): + missing.append("_C.py") + if len(so_files) != 1: + missing.append("exactly one _C_extension*.so") + + print( + f"deepgemm vendored binding: shim={shim.is_file()}, " + f"extensions={[p.name for p in so_files]}" + ) + if missing: + print(f"missing: {missing}", file=sys.stderr) + return 1 + return 0 -def required_pythons() -> list[str]: - pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml" - spec = tomllib.loads(pyproject.read_text())["project"]["requires-python"] - m = re.match(r">=3\.(\d+),<3\.(\d+)", spec) - if not m: - sys.exit(f"unexpected requires-python format: {spec!r}") - return [f"3.{v}" for v in range(int(m[1]), int(m[2]))] - - -spec = importlib.util.find_spec("vllm.third_party.deep_gemm") -if spec is None or spec.origin is None: - sys.exit("vllm.third_party.deep_gemm not importable; is vllm installed?") -pkg_dir = Path(spec.origin).parent - -found = {f"{m[1]}.{m[2]}" for f in os.listdir(pkg_dir) if (m := SO_RE.match(f))} -required = required_pythons() -missing = [v for v in required if v not in found] -print(f"deepgemm _C: found {sorted(found)}, required {required}, missing {missing}") -sys.exit(1 if missing else 0) +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/install_deepgemm.sh b/tools/install_deepgemm.sh index 949aa28da5..2fb2351a0c 100755 --- a/tools/install_deepgemm.sh +++ b/tools/install_deepgemm.sh @@ -10,7 +10,7 @@ DEEPGEMM_GIT_REPO="https://github.com/vllm-project/DeepGEMM.git" # NOTE: This targets the vLLM fork's dev branch, which carries the sm120 # and sm90 paged-MQA ports plus the SwiGLU alpha/beta and SiTU Mega MoE # activations, plus the CUDA 12.x layout header fix from vllm-project/DeepGEMM#12. -DEEPGEMM_GIT_REF="e1f418c2a4f20818221f6b0e578b4c2f634d4c3f" +DEEPGEMM_GIT_REF="1e1842a833699298f7afc02eefb2ed168fff6938" WHEEL_DIR="" # Parse command line arguments @@ -93,6 +93,8 @@ pushd "$INSTALL_DIR/deepgemm" # Checkout the specific reference git checkout "$DEEPGEMM_GIT_REF" +git submodule sync --recursive +git submodule update --init --recursive # Clean previous build artifacts # (Based on https://github.com/deepseek-ai/DeepGEMM/blob/main/install.sh) diff --git a/vllm/model_executor/warmup/deep_gemm_warmup.py b/vllm/model_executor/warmup/deep_gemm_warmup.py index 1266f4ccb2..a4775ce22e 100644 --- a/vllm/model_executor/warmup/deep_gemm_warmup.py +++ b/vllm/model_executor/warmup/deep_gemm_warmup.py @@ -179,6 +179,7 @@ def _deepgemm_fp8_gemm_nt_warmup( device = w.device a1q = torch.empty((max_tokens, k), device=device, dtype=torch.float8_e4m3fn) + # Must be initialized (UE8M0 packing asserts zero sign/mantissa bits). a1q_scales = torch.zeros( (max_tokens, k // block_m), device=device, dtype=torch.float32 ) @@ -284,6 +285,7 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup( ((MAX_M, k // block_m), torch.float32), ((MAX_M, n), torch.bfloat16), ) + # Must be initialized (UE8M0 packing asserts zero sign/mantissa bits). a1q_scales.zero_() for num_tokens, align_used, expert_ids in warmup_cases: