diff --git a/cmake/external_projects/deepgemm.cmake b/cmake/external_projects/deepgemm.cmake index 72f7f5eb66..2ff69bd8b6 100644 --- a/cmake/external_projects/deepgemm.cmake +++ b/cmake/external_projects/deepgemm.cmake @@ -28,9 +28,11 @@ if(DEEPGEMM_SRC_DIR) message(STATUS "DeepGEMM using local DEEPGEMM_SRC_DIR: ${deepgemm_SOURCE_DIR}") else() # Keep in sync with tools/install_deepgemm.sh - set(_DEEPGEMM_UPSTREAM_REPO "https://github.com/deepseek-ai/DeepGEMM.git") - # Pinned to the tip of the nv_dev branch (SM120 support). - set(_DEEPGEMM_UPSTREAM_TAG "8b1392b978f5a03c828dd1711090d7fb50958b8a") + set(_DEEPGEMM_UPSTREAM_REPO "https://github.com/vllm-project/DeepGEMM.git") + # Pinned to the tip of the fork's dev branch: upstream 2.8.0 plus the SM120 + # port, the SM90 paged-MQA kv_block=32/next_n=4 port, configurable SwiGLU + # alpha/beta, and SiTU for FP8/FP4 Mega MoE. + set(_DEEPGEMM_UPSTREAM_TAG "ad1f1726aa540a76c1d26d6a120effb8de21eaa4") set(_deepgemm_fc_root "${FETCHCONTENT_BASE_DIR}") if(NOT _deepgemm_fc_root) @@ -51,7 +53,7 @@ else() BINARY_DIR "${_deepgemm_bin}" GIT_REPOSITORY "${_DEEPGEMM_UPSTREAM_REPO}" GIT_TAG "${_DEEPGEMM_UPSTREAM_TAG}" - GIT_SUBMODULES "third-party/cutlass" "third-party/fmt" + GIT_SUBMODULES "third-party/cutlass" "third-party/deep_jit" GIT_PROGRESS TRUE ) endif() @@ -119,13 +121,15 @@ if(DEEPGEMM_ARCHS) message(STATUS "DeepGEMM _C will be built for: ${_dg_pythons}") # add_custom_command does no implicit header scanning; glob explicitly so - # header-only edits in DeepGEMM/cutlass/fmt re-trigger the rebuild. + # header-only edits in DeepGEMM/cutlass/deep_jit re-trigger the rebuild. file(GLOB_RECURSE _dg_headers "${deepgemm_SOURCE_DIR}/csrc/*.h" "${deepgemm_SOURCE_DIR}/csrc/*.hpp" "${deepgemm_SOURCE_DIR}/deep_gemm/include/*.h" "${deepgemm_SOURCE_DIR}/deep_gemm/include/*.hpp" - "${deepgemm_SOURCE_DIR}/deep_gemm/include/*.cuh") + "${deepgemm_SOURCE_DIR}/deep_gemm/include/*.cuh" + "${deepgemm_SOURCE_DIR}/third-party/deep_jit/include/*.h" + "${deepgemm_SOURCE_DIR}/third-party/deep_jit/include/*.hpp") set(_dg_markers) set(_dg_seen_soabis) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9b344552c5..b3e32777e7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -131,6 +131,7 @@ RUN dnf install -y --setopt=install_weak_deps=False \ git \ curl \ sudo \ + elfutils-devel \ rdma-core-devel \ && dnf clean all \ && rm -rf /var/cache/dnf diff --git a/tools/build_deepgemm_C.py b/tools/build_deepgemm_C.py index 67a527405e..f4438c6755 100644 --- a/tools/build_deepgemm_C.py +++ b/tools/build_deepgemm_C.py @@ -50,7 +50,7 @@ includes = [ str(src / "deep_gemm/include"), str(src / "third-party/cutlass/include"), str(src / "third-party/cutlass/tools/util/include"), - str(src / "third-party/fmt/include"), + str(src / "third-party/deep_jit/include"), *cpp_extension.include_paths(device_type="cuda"), ] diff --git a/tools/install_deepgemm.sh b/tools/install_deepgemm.sh index f6a64d8796..db81bf4be3 100755 --- a/tools/install_deepgemm.sh +++ b/tools/install_deepgemm.sh @@ -6,9 +6,11 @@ set -e # Default values # Keep DEEPGEMM_GIT_REF in sync with cmake/external_projects/deepgemm.cmake -DEEPGEMM_GIT_REPO="https://github.com/deepseek-ai/DeepGEMM.git" -# NOTE: This is currently targeting the nv_dev branch tip due to sm120 support -DEEPGEMM_GIT_REF="8b1392b978f5a03c828dd1711090d7fb50958b8a" +DEEPGEMM_GIT_REPO="https://github.com/vllm-project/DeepGEMM.git" +# NOTE: This targets the vLLM fork's dev branch tip, which carries the sm120 +# and sm90 paged-MQA ports plus the SwiGLU alpha/beta and SiTU Mega MoE +# activations +DEEPGEMM_GIT_REF="ad1f1726aa540a76c1d26d6a120effb8de21eaa4" WHEEL_DIR="" # Parse command line arguments diff --git a/vllm/models/kimi_k3/nvidia/model.py b/vllm/models/kimi_k3/nvidia/model.py index c4f0092f8d..6a26ce78b4 100644 --- a/vllm/models/kimi_k3/nvidia/model.py +++ b/vllm/models/kimi_k3/nvidia/model.py @@ -532,8 +532,10 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts): symm_buffer, activation_clamp=activation_clamp, activation=self.activation, - situ_beta=self.activation_beta, - situ_linear_beta=self.activation_linear_beta, + # DeepGEMM names the SiTU gate tanh scale `activation_alpha` and the + # linear/up tanh scale `activation_beta`; beta=0 leaves up untouched. + activation_alpha=self.activation_beta or 1.0, + activation_beta=self.activation_linear_beta or 0.0, fast_math=fast_math, ) return y diff --git a/vllm/utils/deep_gemm.py b/vllm/utils/deep_gemm.py index 3fc70c0f5d..d515d5131e 100644 --- a/vllm/utils/deep_gemm.py +++ b/vllm/utils/deep_gemm.py @@ -280,7 +280,11 @@ def _lazy_init() -> None: _fp8_gemm_nt_impl = getattr(_dg, "fp8_gemm_nt", None) _fp8_einsum_impl = getattr(_dg, "fp8_einsum", None) _grouped_impl = getattr(_dg, "m_grouped_fp8_gemm_nt_contiguous", None) - _grouped_masked_impl = getattr(_dg, "fp8_m_grouped_gemm_nt_masked", None) + # DeepGEMM 2.8.0 dropped the legacy `fp8_*` aliases; keep both spellings so + # an externally pinned older DeepGEMM still resolves. + _grouped_masked_impl = getattr( + _dg, "m_grouped_fp8_gemm_nt_masked", None + ) or getattr(_dg, "fp8_m_grouped_gemm_nt_masked", None) _grouped_fp4_impl = getattr(_dg, "m_grouped_fp8_fp4_gemm_nt_contiguous", None) # DeepGEMM exposes fp8_fp4_*_mqa_logits as the canonical symbols that # handle both the FP8 and FP4 Q/K paths via a tuple-typed `q`.