[Bugfix] Detect wrong libcute_dsl_runtime.so variant in FlashInfer GDN (#43427)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Qwen3-Next/Qwen3.5 model."""
|
||||
|
||||
import functools
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
@@ -80,6 +82,70 @@ if GDN_AITER_TRITON_AVAILABLE:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# TODO(arpera): remove ``_is_libs_cu13_install_intact`` and its caller in
|
||||
# ``_should_use_flashinfer_gdn_prefill`` once the upstream packaging bug is
|
||||
# fixed and the broken wheels are yanked / superseded on PyPI:
|
||||
# https://github.com/NVIDIA/cutlass/issues/3170
|
||||
# https://github.com/NVIDIA/cutlass/issues/3259
|
||||
@functools.cache
|
||||
def _is_libs_cu13_install_intact() -> bool:
|
||||
"""Return True if every file installed by ``nvidia-cutlass-dsl-libs-cu13``
|
||||
matches the SHA-256 declared in its wheel ``RECORD``.
|
||||
|
||||
``nvidia-cutlass-dsl-libs-base`` and ``nvidia-cutlass-dsl-libs-cu13``
|
||||
both ship into the shared ``nvidia_cutlass_dsl/`` namespace and
|
||||
write many of the same on-disk paths (the runtime ``.so``, the MLIR
|
||||
Python bindings, cuTe-DSL Python sources, ...) with different
|
||||
content. Whichever wheel extracts last wins; with a parallel
|
||||
installer (e.g. ``uv``) the order is racy and the resulting venv
|
||||
can end up with a mix of files from both variants. The
|
||||
``-libs-base`` variant fails MLIR legalization when JIT-compiling
|
||||
the FlashInfer Blackwell GDN prefill kernel, and any other
|
||||
cuTe-DSL-based kernel can break too if on-disk files diverge from
|
||||
what ``-libs-cu13``'s wheel expects. Tracked upstream at:
|
||||
|
||||
* https://github.com/NVIDIA/cutlass/issues/3170
|
||||
* https://github.com/NVIDIA/cutlass/issues/3259
|
||||
|
||||
This helper re-hashes every file the ``-libs-cu13`` wheel claims to
|
||||
own and compares against its declared SHA-256. Returns False on any
|
||||
error (uninstalled, missing RECORD, missing file, hash mismatch).
|
||||
Result is cached per-process.
|
||||
"""
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
|
||||
import pybase64 as base64
|
||||
|
||||
try:
|
||||
dist = importlib.metadata.distribution("nvidia-cutlass-dsl-libs-cu13")
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
return False
|
||||
|
||||
files = dist.files
|
||||
if not files:
|
||||
return False
|
||||
|
||||
for pkg_path in files:
|
||||
file_hash = pkg_path.hash
|
||||
# Skip RECORD rows without a hash (RECORD itself, generated
|
||||
# ``.pyc`` files, ...) and any non-SHA-256 hash modes.
|
||||
if file_hash is None or not file_hash.value:
|
||||
continue
|
||||
if file_hash.mode != "sha256":
|
||||
continue
|
||||
try:
|
||||
with open(pkg_path.locate(), "rb") as f:
|
||||
digest = hashlib.sha256(f.read()).digest()
|
||||
except OSError:
|
||||
return False
|
||||
actual = base64.urlsafe_b64encode(digest).decode().rstrip("=")
|
||||
if actual != file_hash.value:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _should_use_flashinfer_gdn_prefill(backend: str, head_k_dim: int | None) -> bool:
|
||||
"""Whether to use FlashInfer's GDN prefill kernel instead of the
|
||||
Triton/FLA fallback.
|
||||
@@ -89,7 +155,9 @@ def _should_use_flashinfer_gdn_prefill(backend: str, head_k_dim: int | None) ->
|
||||
* ``platform == cuda``;
|
||||
* one of the following:
|
||||
- Hopper (SM90) — no further constraints;
|
||||
- Blackwell (SM10.x) with ``head_k_dim == 128`` and ``cuda_runtime >= 13``.
|
||||
- Blackwell (SM10.x) with ``head_k_dim == 128``, ``cuda_runtime >= 13``,
|
||||
and an intact ``nvidia-cutlass-dsl-libs-cu13`` install on disk
|
||||
(see :func:`_is_libs_cu13_install_intact`).
|
||||
"""
|
||||
if backend not in ["flashinfer", "auto"]:
|
||||
return False
|
||||
@@ -101,7 +169,21 @@ def _should_use_flashinfer_gdn_prefill(backend: str, head_k_dim: int | None) ->
|
||||
return False # Neither Hopper nor Blackwell.
|
||||
if head_k_dim != 128:
|
||||
return False
|
||||
return current_platform.get_cuda_runtime_major() >= 13
|
||||
if current_platform.get_cuda_runtime_major() < 13:
|
||||
return False
|
||||
if not _is_libs_cu13_install_intact():
|
||||
logger.warning_once(
|
||||
"FlashInfer Blackwell GDN requires an intact nvidia-cutlass-dsl"
|
||||
"-libs-cu13 install, but some on-disk files do not match the "
|
||||
"SHA-256 declared in its RECORD (install-order race in "
|
||||
"nvidia-cutlass-dsl packaging — see "
|
||||
"https://github.com/NVIDIA/cutlass/issues/3170 and "
|
||||
"https://github.com/NVIDIA/cutlass/issues/3259). Falling back "
|
||||
"to Triton/FLA. Repair with: pip install --force-reinstall "
|
||||
"--no-deps nvidia-cutlass-dsl-libs-cu13"
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _log_gdn_backend_decision(
|
||||
|
||||
Reference in New Issue
Block a user