[ROCm] Defer tilelang import through its import from vllm.tilelang_utils import tilelang and relaxed has_tilelang (#51159)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fxmarty-amd
2026-08-13 13:12:40 -05:00
committed by GitHub
co-authored by Andreas Karatzas mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent e6b2a8ad5e
commit c5b7c069a4
8 changed files with 313 additions and 52 deletions
Binary file not shown.

Before

Width:  |  Height:  |  Size: 344 KiB

After

Width:  |  Height:  |  Size: 344 KiB

+21
View File
@@ -12,6 +12,7 @@ from unittest import mock
import pytest
from vllm.platforms import current_platform
from vllm.utils import jit_monitor
pytestmark = pytest.mark.cpu_test
@@ -322,6 +323,10 @@ def test_cutedsl_subscripted_compile_is_monitored():
# ------------------------------------------------------------------
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="TileLang JIT monitoring is disabled on ROCm",
)
def test_tilelang_jit_kernel_logs_warning():
with _patch_jit_modules(_make_fake_knobs()):
from tilelang.jit.kernel import JITKernel
@@ -337,6 +342,10 @@ def test_tilelang_jit_kernel_logs_warning():
assert "tl_kernel" in msg
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="TileLang JIT monitoring is disabled on ROCm",
)
def test_tilelang_jit_impl_logs_warning():
with _patch_jit_modules(_make_fake_knobs()):
from tilelang.jit import JITImpl
@@ -386,6 +395,10 @@ def test_tilelang_jit_impl_logs_warning():
assert "tilelang_fn" in msg
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="TileLang JIT monitoring is disabled on ROCm",
)
def test_tilelang_jit_impl_does_not_log_on_cache_hit():
with _patch_jit_modules(_make_fake_knobs()):
from tilelang.jit import JITImpl
@@ -413,6 +426,10 @@ def test_tilelang_jit_impl_does_not_log_on_cache_hit():
warning_once.assert_called_once()
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="TileLang JIT monitoring is disabled on ROCm",
)
def test_tilelang_from_database_does_not_log():
with _patch_jit_modules(_make_fake_knobs()):
from tilelang.jit.kernel import JITKernel
@@ -425,6 +442,10 @@ def test_tilelang_from_database_does_not_log():
warning_once.assert_not_called()
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="TileLang JIT monitoring is disabled on ROCm",
)
def test_tilelang_error_mode_raises():
with _patch_jit_modules(_make_fake_knobs()):
from tilelang.jit.kernel import JITKernel
+128
View File
@@ -0,0 +1,128 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import ctypes
import importlib
import importlib.util
import sys
from types import ModuleType
from typing import Any
import pytest
from vllm.platforms import current_platform
from vllm.utils import import_utils
class _PassConfigKey:
TL_DISABLE_WARP_SPECIALIZED = "disable_warp_specialized"
TL_DISABLE_TMA_LOWER = "disable_tma_lower"
TL_PTXAS_REGISTER_USAGE_LEVEL = "ptxas_register_usage_level"
def _install_tilelang_stub(
monkeypatch: pytest.MonkeyPatch,
) -> dict[str, int]:
calls = {"jit_decorate": 0, "compiled_call": 0}
tilelang: Any = ModuleType("tilelang")
def jit(**kwargs: Any) -> Any:
def decorate(func: Any) -> Any:
calls["jit_decorate"] += 1
def compiled(*args: Any, **kw: Any) -> Any:
calls["compiled_call"] += 1
return func.__name__
return compiled
return decorate
tilelang.PassConfigKey = _PassConfigKey
tilelang.jit = jit
monkeypatch.setattr(import_utils, "has_tilelang", lambda: True)
monkeypatch.setitem(sys.modules, "tilelang", tilelang)
monkeypatch.setitem(
sys.modules, "tilelang.language", ModuleType("tilelang.language")
)
monkeypatch.delitem(sys.modules, "vllm.tilelang_utils", raising=False)
return calls
def test_tilelang_jit_decorator_is_lazy_only_on_rocm(
monkeypatch: pytest.MonkeyPatch,
) -> None:
if not (current_platform.is_cuda() or current_platform.is_rocm()):
pytest.skip("Test requires CUDA or ROCm")
calls = _install_tilelang_stub(monkeypatch)
module_name = "vllm.model_executor.kernels.mhc.tilelang_kernels"
monkeypatch.delitem(sys.modules, module_name, raising=False)
module = importlib.import_module(module_name)
if current_platform.is_rocm():
assert calls["jit_decorate"] == 0
else:
assert calls["jit_decorate"] > 0
decorated_calls = calls["jit_decorate"]
assert module.mhc_post_tilelang() == "mhc_post_tilelang"
if current_platform.is_rocm():
assert calls["jit_decorate"] == 1
else:
assert calls["jit_decorate"] == decorated_calls
assert calls["compiled_call"] == 1
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Test requires ROCm")
def test_deepseek_v4_import_and_jit_monitor_do_not_hijack_hip_symbols(
monkeypatch: pytest.MonkeyPatch,
) -> None:
if importlib.util.find_spec("tilelang") is None:
pytest.skip("Test requires TileLang to be installed")
class DlInfo(ctypes.Structure):
_fields_ = [
("dli_fname", ctypes.c_char_p),
("dli_fbase", ctypes.c_void_p),
("dli_sname", ctypes.c_char_p),
("dli_saddr", ctypes.c_void_p),
]
libdl = ctypes.CDLL("libdl.so.2")
dlsym = libdl.dlsym
dlsym.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
dlsym.restype = ctypes.c_void_p
dladdr = libdl.dladdr
dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(DlInfo)]
dladdr.restype = ctypes.c_int
from vllm.model_executor.layers import mhc # noqa: F401
from vllm.models import deepseek_v4 # noqa: F401
from vllm.utils import jit_monitor
monkeypatch.setattr(jit_monitor, "_active", False)
monkeypatch.setattr(jit_monitor, "_setup_triton_autotuning_print", lambda: None)
monkeypatch.setattr(jit_monitor, "_setup_triton_jit_hook", lambda: None)
monkeypatch.setattr(jit_monitor, "_setup_cutedsl_jit_hook", lambda: None)
monkeypatch.setattr(
jit_monitor,
"_setup_tilelang_jit_hook",
lambda: pytest.fail("TileLang JIT monitor must not run on ROCm"),
)
jit_monitor.activate()
assert not any(
name == "tilelang" or name.startswith("tilelang.") for name in sys.modules
)
address = dlsym(None, b"hipFree")
assert address is not None, "hipFree is not available in the global symbol table"
info = DlInfo()
assert dladdr(address, ctypes.byref(info))
source = info.dli_fname.decode() if info.dli_fname else "<unknown>"
assert "libhip_stub.so" not in source, source
assert "libamdhip64.so" in source, source
@@ -92,6 +92,15 @@ CHECK_IMPORTS = {
),
allowed_files={"vllm/triton_utils/importing.py"},
),
"tilelang": ForbiddenImport(
pattern=r"^(from|import)\s+tilelang(\s|\.|$)",
tip="Use 'from vllm.tilelang_utils import tilelang, T' instead.",
allowed_pattern=re.compile(
r"from\s+vllm\.tilelang_utils\s+import\s+"
r"(tilelang|T|T, tilelang|tilelang, T)\b"
),
allowed_files={"vllm/tilelang_utils/__init__.py"},
),
"huggingface_hub repo API": ForbiddenImport(
# Catch `from huggingface_hub import <fn>`, including parenthesized,
# multi-line imports.
@@ -182,6 +191,29 @@ def test_regex():
f"(expected {should_match}, got {result})"
)
tilelang_cases = [
# Should match
("import tilelang", True),
("import tilelang.language as T", True),
("from tilelang.jit import JITImpl", True),
("from tilelang.jit.kernel import JITKernel", True),
# Should not match: indented (local) imports are allowed, mirroring
# the "triton" rule, so mocked-module test imports are not flagged.
(" import tilelang", False),
(" from tilelang.jit import JITImpl", False),
("from vllm.tilelang_utils import tilelang", False),
("from vllm.tilelang_utils import T", False),
("from vllm.tilelang_utils import T, tilelang", False),
("from vllm.tilelang_utils import tilelang, T", False),
("import tilelang_kernels", False),
]
for i, (content, should_match) in enumerate(tilelang_cases):
result = matches("tilelang", content)
assert result == should_match, (
f"tilelang case {i} failed: {content!r} "
f"(expected {should_match}, got {result})"
)
hf_cases = [
# Should match
("from huggingface_hub import snapshot_download", True),
@@ -1,29 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import math
from functools import cache
from typing import TYPE_CHECKING, Any
import torch
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_tilelang
from vllm.tilelang_utils import T, tilelang, tilelang_jit
from vllm.utils.math_utils import cdiv
# TileLang is used for MHC on CUDA and ROCm. Keep non-GPU imports cheap so
# registering the Python wrapper modules does not require TileLang everywhere.
if TYPE_CHECKING or current_platform.is_cuda_alike():
if not has_tilelang():
raise ImportError(
"tilelang is required for mhc but is not installed. Install it with "
"`pip install tilelang`."
)
import tilelang
import tilelang.language as T
else:
tilelang = None # type: ignore[assignment]
T = None # type: ignore[assignment]
ENABLE_PDL = current_platform.is_arch_support_pdl() and current_platform.is_cuda()
@@ -40,18 +28,7 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int:
return split_k
pass_configs: dict[tilelang.PassConfigKey, Any] = {
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
}
if current_platform.is_cuda():
pass_configs[tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL] = 10
@tilelang.jit(
pass_configs=pass_configs,
)
@tilelang_jit
def mhc_pre_big_fuse_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
@@ -191,9 +168,7 @@ def mhc_pre_big_fuse_tilelang(
# Copied from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/mhc.py#L478
@tilelang.jit(
pass_configs=pass_configs,
)
@tilelang_jit
def mhc_pre_big_fuse_with_norm_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
@@ -353,9 +328,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
T.pdl_trigger()
@tilelang.jit(
pass_configs=pass_configs,
)
@tilelang_jit
def mhc_pre_big_fuse_broadcast_with_norm_tilelang(
gemm_out_mul,
gemm_out_sqrsum,
@@ -517,9 +490,7 @@ def mhc_pre_big_fuse_broadcast_with_norm_tilelang(
T.pdl_trigger()
@tilelang.jit(
pass_configs=pass_configs,
)
@tilelang_jit
def mhc_fused_tilelang(
comb_mix,
residual_in,
@@ -640,9 +611,7 @@ def mhc_fused_tilelang(
T.pdl_trigger()
@tilelang.jit(
pass_configs=pass_configs,
)
@tilelang_jit
def mhc_post_tilelang(
a,
b,
@@ -695,9 +664,7 @@ def mhc_post_tilelang(
T.pdl_trigger()
@tilelang.jit(
pass_configs=pass_configs,
)
@tilelang_jit
def hc_prenorm_gemm_tilelang(
x,
fn,
@@ -781,9 +748,7 @@ def hc_prenorm_gemm_tilelang(
T.pdl_trigger()
@tilelang.jit(
pass_configs=pass_configs,
)
@tilelang_jit
def hc_prenorm_gemm_block_m_tilelang(
x,
fn,
@@ -878,9 +843,7 @@ def hc_prenorm_gemm_block_m_tilelang(
T.pdl_trigger()
@tilelang.jit(
pass_configs=pass_configs,
)
@tilelang_jit
def hc_head_fuse_tilelang(
residual,
fn,
+91
View File
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import functools
from collections.abc import Callable
from functools import cache
from typing import TYPE_CHECKING, Any
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_tilelang
if TYPE_CHECKING or current_platform.is_cuda():
if not has_tilelang():
raise ImportError(
"tilelang is required for mhc but is not installed. Install it with "
"`pip install tilelang`."
)
import tilelang
import tilelang.language as T
else:
tilelang = None # type: ignore[assignment]
T = None # type: ignore[assignment]
def _ensure_tilelang_imported() -> None:
"""Bind the `tilelang` and `T` module globals, importing them if needed.
On ROCm, this runs on the first kernel call instead of at import time.
Raises:
ImportError: If TileLang is not installed.
"""
global T, tilelang
if tilelang is not None:
return
if not has_tilelang():
raise ImportError(
"tilelang is required for mhc but is not installed. Install it with "
"`pip install tilelang`."
)
import tilelang as tilelang_module
import tilelang.language as tilelang_language
tilelang = tilelang_module
T = tilelang_language
@cache
def _get_pass_configs() -> dict[Any, Any]:
_ensure_tilelang_imported()
pass_configs: dict[Any, Any] = {
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
}
if current_platform.is_cuda():
pass_configs[tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL] = 10
return pass_configs
def tilelang_jit(kernel_function: Callable[..., Any]) -> Callable[..., Any]:
"""Apply `tilelang.jit`, deferring until first call on ROCm.
ROCm defers JIT decoration so importing the caller's module does not
require TileLang immediately. CUDA keeps the eager decoration behavior.
The kernel body parsed by TileLang references `T` as an unqualified
global, so on the deferred ROCm path this rebinds `T`/`tilelang` in the
decorated function's own module globals once they become available.
"""
if not current_platform.is_rocm():
_ensure_tilelang_imported()
return tilelang.jit(pass_configs=_get_pass_configs())(kernel_function)
compiled_kernel: Callable[..., Any] | None = None
@functools.wraps(kernel_function)
def wrapper(*args: Any, **kwargs: Any) -> Any:
nonlocal compiled_kernel
if compiled_kernel is None:
_ensure_tilelang_imported()
kernel_function.__globals__["tilelang"] = tilelang
kernel_function.__globals__["T"] = T
compiled_kernel = tilelang.jit(pass_configs=_get_pass_configs())(
kernel_function
)
return compiled_kernel(*args, **kwargs)
return wrapper
+22 -2
View File
@@ -412,6 +412,21 @@ def _has_module(module_name: str) -> bool:
return True
@cache
def _has_module_spec(module_name: str) -> bool:
"""Return True if *module_name* is installed, without importing it.
Unlike [`_has_module`][vllm.utils.import_utils._has_module], this only
resolves the import spec. It therefore does not pay the import cost of
heavyweight modules, at the price of not verifying that native
dependencies (shared libraries, etc.) are satisfied. The result is cached.
"""
try:
return importlib.util.find_spec(module_name) is not None
except Exception:
return False
def has_deep_ep() -> bool:
"""Whether the optional `deep_ep` package is available."""
return _has_module("deep_ep")
@@ -504,8 +519,13 @@ def has_triton_kernels() -> bool:
@cache
def has_tilelang() -> bool:
"""Whether the optional `tilelang` package is available."""
if not _has_module("tilelang"):
"""Whether the optional `tilelang` package is available.
Only the import spec is checked: importing `tilelang` is expensive, so
callers must import it lazily at their point of use rather than relying
on this function to have imported it already.
"""
if not _has_module_spec("tilelang"):
return False
# ROCm-only guard, imported lazily to avoid loading rocm on CUDA.
from vllm.platforms import current_platform
+7 -1
View File
@@ -30,6 +30,7 @@ from contextlib import suppress
from typing import Any, Literal, cast
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.triton_utils.importing import HAS_TRITON
logger = init_logger(__name__)
@@ -74,7 +75,12 @@ def activate(*, mode: JitMonitorMode = "warn", verbose: bool = False) -> None:
_setup_triton_autotuning_print()
_setup_triton_jit_hook()
_setup_cutedsl_jit_hook()
_setup_tilelang_jit_hook()
# Refer to #51159. tilelang ships broken symbols
# on rocm.
# TODO: Remove the guard once tilelang upstream is fixed.
if not current_platform.is_rocm():
_setup_tilelang_jit_hook()
logger.info(
"Kernel JIT monitor activated; monitored JIT compilations during "