diff --git a/docs/assets/contributing/dockerfile-stages-dependency.png b/docs/assets/contributing/dockerfile-stages-dependency.png index 510bf7e2b1..be2a4cc4f3 100644 Binary files a/docs/assets/contributing/dockerfile-stages-dependency.png and b/docs/assets/contributing/dockerfile-stages-dependency.png differ diff --git a/tests/jit_monitor/test_hooks.py b/tests/jit_monitor/test_hooks.py index 1285b4c89e..48cd9fd0f6 100644 --- a/tests/jit_monitor/test_hooks.py +++ b/tests/jit_monitor/test_hooks.py @@ -12,6 +12,7 @@ from unittest import mock import pytest +from vllm.platforms import current_platform from vllm.utils import jit_monitor pytestmark = pytest.mark.cpu_test @@ -322,6 +323,10 @@ def test_cutedsl_subscripted_compile_is_monitored(): # ------------------------------------------------------------------ +@pytest.mark.skipif( + current_platform.is_rocm(), + reason="TileLang JIT monitoring is disabled on ROCm", +) def test_tilelang_jit_kernel_logs_warning(): with _patch_jit_modules(_make_fake_knobs()): from tilelang.jit.kernel import JITKernel @@ -337,6 +342,10 @@ def test_tilelang_jit_kernel_logs_warning(): assert "tl_kernel" in msg +@pytest.mark.skipif( + current_platform.is_rocm(), + reason="TileLang JIT monitoring is disabled on ROCm", +) def test_tilelang_jit_impl_logs_warning(): with _patch_jit_modules(_make_fake_knobs()): from tilelang.jit import JITImpl @@ -386,6 +395,10 @@ def test_tilelang_jit_impl_logs_warning(): assert "tilelang_fn" in msg +@pytest.mark.skipif( + current_platform.is_rocm(), + reason="TileLang JIT monitoring is disabled on ROCm", +) def test_tilelang_jit_impl_does_not_log_on_cache_hit(): with _patch_jit_modules(_make_fake_knobs()): from tilelang.jit import JITImpl @@ -413,6 +426,10 @@ def test_tilelang_jit_impl_does_not_log_on_cache_hit(): warning_once.assert_called_once() +@pytest.mark.skipif( + current_platform.is_rocm(), + reason="TileLang JIT monitoring is disabled on ROCm", +) def test_tilelang_from_database_does_not_log(): with _patch_jit_modules(_make_fake_knobs()): from tilelang.jit.kernel import JITKernel @@ -425,6 +442,10 @@ def test_tilelang_from_database_does_not_log(): warning_once.assert_not_called() +@pytest.mark.skipif( + current_platform.is_rocm(), + reason="TileLang JIT monitoring is disabled on ROCm", +) def test_tilelang_error_mode_raises(): with _patch_jit_modules(_make_fake_knobs()): from tilelang.jit.kernel import JITKernel diff --git a/tests/kernels/test_mhc_tilelang_jit.py b/tests/kernels/test_mhc_tilelang_jit.py new file mode 100644 index 0000000000..15560165ce --- /dev/null +++ b/tests/kernels/test_mhc_tilelang_jit.py @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import ctypes +import importlib +import importlib.util +import sys +from types import ModuleType +from typing import Any + +import pytest + +from vllm.platforms import current_platform +from vllm.utils import import_utils + + +class _PassConfigKey: + TL_DISABLE_WARP_SPECIALIZED = "disable_warp_specialized" + TL_DISABLE_TMA_LOWER = "disable_tma_lower" + TL_PTXAS_REGISTER_USAGE_LEVEL = "ptxas_register_usage_level" + + +def _install_tilelang_stub( + monkeypatch: pytest.MonkeyPatch, +) -> dict[str, int]: + calls = {"jit_decorate": 0, "compiled_call": 0} + + tilelang: Any = ModuleType("tilelang") + + def jit(**kwargs: Any) -> Any: + def decorate(func: Any) -> Any: + calls["jit_decorate"] += 1 + + def compiled(*args: Any, **kw: Any) -> Any: + calls["compiled_call"] += 1 + return func.__name__ + + return compiled + + return decorate + + tilelang.PassConfigKey = _PassConfigKey + tilelang.jit = jit + + monkeypatch.setattr(import_utils, "has_tilelang", lambda: True) + monkeypatch.setitem(sys.modules, "tilelang", tilelang) + monkeypatch.setitem( + sys.modules, "tilelang.language", ModuleType("tilelang.language") + ) + monkeypatch.delitem(sys.modules, "vllm.tilelang_utils", raising=False) + + return calls + + +def test_tilelang_jit_decorator_is_lazy_only_on_rocm( + monkeypatch: pytest.MonkeyPatch, +) -> None: + if not (current_platform.is_cuda() or current_platform.is_rocm()): + pytest.skip("Test requires CUDA or ROCm") + + calls = _install_tilelang_stub(monkeypatch) + module_name = "vllm.model_executor.kernels.mhc.tilelang_kernels" + monkeypatch.delitem(sys.modules, module_name, raising=False) + module = importlib.import_module(module_name) + + if current_platform.is_rocm(): + assert calls["jit_decorate"] == 0 + else: + assert calls["jit_decorate"] > 0 + + decorated_calls = calls["jit_decorate"] + assert module.mhc_post_tilelang() == "mhc_post_tilelang" + if current_platform.is_rocm(): + assert calls["jit_decorate"] == 1 + else: + assert calls["jit_decorate"] == decorated_calls + assert calls["compiled_call"] == 1 + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="Test requires ROCm") +def test_deepseek_v4_import_and_jit_monitor_do_not_hijack_hip_symbols( + monkeypatch: pytest.MonkeyPatch, +) -> None: + if importlib.util.find_spec("tilelang") is None: + pytest.skip("Test requires TileLang to be installed") + + class DlInfo(ctypes.Structure): + _fields_ = [ + ("dli_fname", ctypes.c_char_p), + ("dli_fbase", ctypes.c_void_p), + ("dli_sname", ctypes.c_char_p), + ("dli_saddr", ctypes.c_void_p), + ] + + libdl = ctypes.CDLL("libdl.so.2") + dlsym = libdl.dlsym + dlsym.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + dlsym.restype = ctypes.c_void_p + dladdr = libdl.dladdr + dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(DlInfo)] + dladdr.restype = ctypes.c_int + + from vllm.model_executor.layers import mhc # noqa: F401 + from vllm.models import deepseek_v4 # noqa: F401 + from vllm.utils import jit_monitor + + monkeypatch.setattr(jit_monitor, "_active", False) + monkeypatch.setattr(jit_monitor, "_setup_triton_autotuning_print", lambda: None) + monkeypatch.setattr(jit_monitor, "_setup_triton_jit_hook", lambda: None) + monkeypatch.setattr(jit_monitor, "_setup_cutedsl_jit_hook", lambda: None) + monkeypatch.setattr( + jit_monitor, + "_setup_tilelang_jit_hook", + lambda: pytest.fail("TileLang JIT monitor must not run on ROCm"), + ) + jit_monitor.activate() + + assert not any( + name == "tilelang" or name.startswith("tilelang.") for name in sys.modules + ) + + address = dlsym(None, b"hipFree") + assert address is not None, "hipFree is not available in the global symbol table" + info = DlInfo() + assert dladdr(address, ctypes.byref(info)) + source = info.dli_fname.decode() if info.dli_fname else "" + assert "libhip_stub.so" not in source, source + assert "libamdhip64.so" in source, source diff --git a/tools/pre_commit/check_forbidden_imports.py b/tools/pre_commit/check_forbidden_imports.py index 52a95ce1d8..59163a23e0 100644 --- a/tools/pre_commit/check_forbidden_imports.py +++ b/tools/pre_commit/check_forbidden_imports.py @@ -92,6 +92,15 @@ CHECK_IMPORTS = { ), allowed_files={"vllm/triton_utils/importing.py"}, ), + "tilelang": ForbiddenImport( + pattern=r"^(from|import)\s+tilelang(\s|\.|$)", + tip="Use 'from vllm.tilelang_utils import tilelang, T' instead.", + allowed_pattern=re.compile( + r"from\s+vllm\.tilelang_utils\s+import\s+" + r"(tilelang|T|T, tilelang|tilelang, T)\b" + ), + allowed_files={"vllm/tilelang_utils/__init__.py"}, + ), "huggingface_hub repo API": ForbiddenImport( # Catch `from huggingface_hub import `, including parenthesized, # multi-line imports. @@ -182,6 +191,29 @@ def test_regex(): f"(expected {should_match}, got {result})" ) + tilelang_cases = [ + # Should match + ("import tilelang", True), + ("import tilelang.language as T", True), + ("from tilelang.jit import JITImpl", True), + ("from tilelang.jit.kernel import JITKernel", True), + # Should not match: indented (local) imports are allowed, mirroring + # the "triton" rule, so mocked-module test imports are not flagged. + (" import tilelang", False), + (" from tilelang.jit import JITImpl", False), + ("from vllm.tilelang_utils import tilelang", False), + ("from vllm.tilelang_utils import T", False), + ("from vllm.tilelang_utils import T, tilelang", False), + ("from vllm.tilelang_utils import tilelang, T", False), + ("import tilelang_kernels", False), + ] + for i, (content, should_match) in enumerate(tilelang_cases): + result = matches("tilelang", content) + assert result == should_match, ( + f"tilelang case {i} failed: {content!r} " + f"(expected {should_match}, got {result})" + ) + hf_cases = [ # Should match ("from huggingface_hub import snapshot_download", True), diff --git a/vllm/model_executor/kernels/mhc/tilelang_kernels.py b/vllm/model_executor/kernels/mhc/tilelang_kernels.py index 3afd7069aa..925f4631a5 100644 --- a/vllm/model_executor/kernels/mhc/tilelang_kernels.py +++ b/vllm/model_executor/kernels/mhc/tilelang_kernels.py @@ -1,29 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + import math from functools import cache -from typing import TYPE_CHECKING, Any import torch from vllm.platforms import current_platform -from vllm.utils.import_utils import has_tilelang +from vllm.tilelang_utils import T, tilelang, tilelang_jit from vllm.utils.math_utils import cdiv -# TileLang is used for MHC on CUDA and ROCm. Keep non-GPU imports cheap so -# registering the Python wrapper modules does not require TileLang everywhere. -if TYPE_CHECKING or current_platform.is_cuda_alike(): - if not has_tilelang(): - raise ImportError( - "tilelang is required for mhc but is not installed. Install it with " - "`pip install tilelang`." - ) - import tilelang - import tilelang.language as T -else: - tilelang = None # type: ignore[assignment] - T = None # type: ignore[assignment] - ENABLE_PDL = current_platform.is_arch_support_pdl() and current_platform.is_cuda() @@ -40,18 +28,7 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int: return split_k -pass_configs: dict[tilelang.PassConfigKey, Any] = { - tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, - tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, -} - -if current_platform.is_cuda(): - pass_configs[tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL] = 10 - - -@tilelang.jit( - pass_configs=pass_configs, -) +@tilelang_jit def mhc_pre_big_fuse_tilelang( gemm_out_mul, gemm_out_sqrsum, @@ -191,9 +168,7 @@ def mhc_pre_big_fuse_tilelang( # Copied from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/mhc.py#L478 -@tilelang.jit( - pass_configs=pass_configs, -) +@tilelang_jit def mhc_pre_big_fuse_with_norm_tilelang( gemm_out_mul, gemm_out_sqrsum, @@ -353,9 +328,7 @@ def mhc_pre_big_fuse_with_norm_tilelang( T.pdl_trigger() -@tilelang.jit( - pass_configs=pass_configs, -) +@tilelang_jit def mhc_pre_big_fuse_broadcast_with_norm_tilelang( gemm_out_mul, gemm_out_sqrsum, @@ -517,9 +490,7 @@ def mhc_pre_big_fuse_broadcast_with_norm_tilelang( T.pdl_trigger() -@tilelang.jit( - pass_configs=pass_configs, -) +@tilelang_jit def mhc_fused_tilelang( comb_mix, residual_in, @@ -640,9 +611,7 @@ def mhc_fused_tilelang( T.pdl_trigger() -@tilelang.jit( - pass_configs=pass_configs, -) +@tilelang_jit def mhc_post_tilelang( a, b, @@ -695,9 +664,7 @@ def mhc_post_tilelang( T.pdl_trigger() -@tilelang.jit( - pass_configs=pass_configs, -) +@tilelang_jit def hc_prenorm_gemm_tilelang( x, fn, @@ -781,9 +748,7 @@ def hc_prenorm_gemm_tilelang( T.pdl_trigger() -@tilelang.jit( - pass_configs=pass_configs, -) +@tilelang_jit def hc_prenorm_gemm_block_m_tilelang( x, fn, @@ -878,9 +843,7 @@ def hc_prenorm_gemm_block_m_tilelang( T.pdl_trigger() -@tilelang.jit( - pass_configs=pass_configs, -) +@tilelang_jit def hc_head_fuse_tilelang( residual, fn, diff --git a/vllm/tilelang_utils/__init__.py b/vllm/tilelang_utils/__init__.py new file mode 100644 index 0000000000..803409b1ec --- /dev/null +++ b/vllm/tilelang_utils/__init__.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import functools +from collections.abc import Callable +from functools import cache +from typing import TYPE_CHECKING, Any + +from vllm.platforms import current_platform +from vllm.utils.import_utils import has_tilelang + +if TYPE_CHECKING or current_platform.is_cuda(): + if not has_tilelang(): + raise ImportError( + "tilelang is required for mhc but is not installed. Install it with " + "`pip install tilelang`." + ) + import tilelang + import tilelang.language as T +else: + tilelang = None # type: ignore[assignment] + T = None # type: ignore[assignment] + + +def _ensure_tilelang_imported() -> None: + """Bind the `tilelang` and `T` module globals, importing them if needed. + + On ROCm, this runs on the first kernel call instead of at import time. + + Raises: + ImportError: If TileLang is not installed. + """ + global T, tilelang + + if tilelang is not None: + return + if not has_tilelang(): + raise ImportError( + "tilelang is required for mhc but is not installed. Install it with " + "`pip install tilelang`." + ) + import tilelang as tilelang_module + import tilelang.language as tilelang_language + + tilelang = tilelang_module + T = tilelang_language + + +@cache +def _get_pass_configs() -> dict[Any, Any]: + _ensure_tilelang_imported() + pass_configs: dict[Any, Any] = { + tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, + tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, + } + if current_platform.is_cuda(): + pass_configs[tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL] = 10 + return pass_configs + + +def tilelang_jit(kernel_function: Callable[..., Any]) -> Callable[..., Any]: + """Apply `tilelang.jit`, deferring until first call on ROCm. + + ROCm defers JIT decoration so importing the caller's module does not + require TileLang immediately. CUDA keeps the eager decoration behavior. + + The kernel body parsed by TileLang references `T` as an unqualified + global, so on the deferred ROCm path this rebinds `T`/`tilelang` in the + decorated function's own module globals once they become available. + """ + if not current_platform.is_rocm(): + _ensure_tilelang_imported() + return tilelang.jit(pass_configs=_get_pass_configs())(kernel_function) + + compiled_kernel: Callable[..., Any] | None = None + + @functools.wraps(kernel_function) + def wrapper(*args: Any, **kwargs: Any) -> Any: + nonlocal compiled_kernel + if compiled_kernel is None: + _ensure_tilelang_imported() + kernel_function.__globals__["tilelang"] = tilelang + kernel_function.__globals__["T"] = T + compiled_kernel = tilelang.jit(pass_configs=_get_pass_configs())( + kernel_function + ) + return compiled_kernel(*args, **kwargs) + + return wrapper diff --git a/vllm/utils/import_utils.py b/vllm/utils/import_utils.py index 9f6ea0fec4..03e301317f 100644 --- a/vllm/utils/import_utils.py +++ b/vllm/utils/import_utils.py @@ -412,6 +412,21 @@ def _has_module(module_name: str) -> bool: return True +@cache +def _has_module_spec(module_name: str) -> bool: + """Return True if *module_name* is installed, without importing it. + + Unlike [`_has_module`][vllm.utils.import_utils._has_module], this only + resolves the import spec. It therefore does not pay the import cost of + heavyweight modules, at the price of not verifying that native + dependencies (shared libraries, etc.) are satisfied. The result is cached. + """ + try: + return importlib.util.find_spec(module_name) is not None + except Exception: + return False + + def has_deep_ep() -> bool: """Whether the optional `deep_ep` package is available.""" return _has_module("deep_ep") @@ -504,8 +519,13 @@ def has_triton_kernels() -> bool: @cache def has_tilelang() -> bool: - """Whether the optional `tilelang` package is available.""" - if not _has_module("tilelang"): + """Whether the optional `tilelang` package is available. + + Only the import spec is checked: importing `tilelang` is expensive, so + callers must import it lazily at their point of use rather than relying + on this function to have imported it already. + """ + if not _has_module_spec("tilelang"): return False # ROCm-only guard, imported lazily to avoid loading rocm on CUDA. from vllm.platforms import current_platform diff --git a/vllm/utils/jit_monitor.py b/vllm/utils/jit_monitor.py index cc823f704b..bb0bb8df9a 100644 --- a/vllm/utils/jit_monitor.py +++ b/vllm/utils/jit_monitor.py @@ -30,6 +30,7 @@ from contextlib import suppress from typing import Any, Literal, cast from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.triton_utils.importing import HAS_TRITON logger = init_logger(__name__) @@ -74,7 +75,12 @@ def activate(*, mode: JitMonitorMode = "warn", verbose: bool = False) -> None: _setup_triton_autotuning_print() _setup_triton_jit_hook() _setup_cutedsl_jit_hook() - _setup_tilelang_jit_hook() + + # Refer to #51159. tilelang ships broken symbols + # on rocm. + # TODO: Remove the guard once tilelang upstream is fixed. + if not current_platform.is_rocm(): + _setup_tilelang_jit_hook() logger.info( "Kernel JIT monitor activated; monitored JIT compilations during "