[ROCm] Defer tilelang import through its import from vllm.tilelang_utils import tilelang and relaxed has_tilelang (#51159)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Andreas Karatzas
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e6b2a8ad5e
commit
c5b7c069a4
Binary file not shown.
|
Before Width: | Height: | Size: 344 KiB After Width: | Height: | Size: 344 KiB |
@@ -12,6 +12,7 @@ from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import jit_monitor
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -322,6 +323,10 @@ def test_cutedsl_subscripted_compile_is_monitored():
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="TileLang JIT monitoring is disabled on ROCm",
|
||||
)
|
||||
def test_tilelang_jit_kernel_logs_warning():
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
from tilelang.jit.kernel import JITKernel
|
||||
@@ -337,6 +342,10 @@ def test_tilelang_jit_kernel_logs_warning():
|
||||
assert "tl_kernel" in msg
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="TileLang JIT monitoring is disabled on ROCm",
|
||||
)
|
||||
def test_tilelang_jit_impl_logs_warning():
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
from tilelang.jit import JITImpl
|
||||
@@ -386,6 +395,10 @@ def test_tilelang_jit_impl_logs_warning():
|
||||
assert "tilelang_fn" in msg
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="TileLang JIT monitoring is disabled on ROCm",
|
||||
)
|
||||
def test_tilelang_jit_impl_does_not_log_on_cache_hit():
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
from tilelang.jit import JITImpl
|
||||
@@ -413,6 +426,10 @@ def test_tilelang_jit_impl_does_not_log_on_cache_hit():
|
||||
warning_once.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="TileLang JIT monitoring is disabled on ROCm",
|
||||
)
|
||||
def test_tilelang_from_database_does_not_log():
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
from tilelang.jit.kernel import JITKernel
|
||||
@@ -425,6 +442,10 @@ def test_tilelang_from_database_does_not_log():
|
||||
warning_once.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="TileLang JIT monitoring is disabled on ROCm",
|
||||
)
|
||||
def test_tilelang_error_mode_raises():
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
from tilelang.jit.kernel import JITKernel
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import ctypes
|
||||
import importlib
|
||||
import importlib.util
|
||||
import sys
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import import_utils
|
||||
|
||||
|
||||
class _PassConfigKey:
|
||||
TL_DISABLE_WARP_SPECIALIZED = "disable_warp_specialized"
|
||||
TL_DISABLE_TMA_LOWER = "disable_tma_lower"
|
||||
TL_PTXAS_REGISTER_USAGE_LEVEL = "ptxas_register_usage_level"
|
||||
|
||||
|
||||
def _install_tilelang_stub(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> dict[str, int]:
|
||||
calls = {"jit_decorate": 0, "compiled_call": 0}
|
||||
|
||||
tilelang: Any = ModuleType("tilelang")
|
||||
|
||||
def jit(**kwargs: Any) -> Any:
|
||||
def decorate(func: Any) -> Any:
|
||||
calls["jit_decorate"] += 1
|
||||
|
||||
def compiled(*args: Any, **kw: Any) -> Any:
|
||||
calls["compiled_call"] += 1
|
||||
return func.__name__
|
||||
|
||||
return compiled
|
||||
|
||||
return decorate
|
||||
|
||||
tilelang.PassConfigKey = _PassConfigKey
|
||||
tilelang.jit = jit
|
||||
|
||||
monkeypatch.setattr(import_utils, "has_tilelang", lambda: True)
|
||||
monkeypatch.setitem(sys.modules, "tilelang", tilelang)
|
||||
monkeypatch.setitem(
|
||||
sys.modules, "tilelang.language", ModuleType("tilelang.language")
|
||||
)
|
||||
monkeypatch.delitem(sys.modules, "vllm.tilelang_utils", raising=False)
|
||||
|
||||
return calls
|
||||
|
||||
|
||||
def test_tilelang_jit_decorator_is_lazy_only_on_rocm(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
if not (current_platform.is_cuda() or current_platform.is_rocm()):
|
||||
pytest.skip("Test requires CUDA or ROCm")
|
||||
|
||||
calls = _install_tilelang_stub(monkeypatch)
|
||||
module_name = "vllm.model_executor.kernels.mhc.tilelang_kernels"
|
||||
monkeypatch.delitem(sys.modules, module_name, raising=False)
|
||||
module = importlib.import_module(module_name)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
assert calls["jit_decorate"] == 0
|
||||
else:
|
||||
assert calls["jit_decorate"] > 0
|
||||
|
||||
decorated_calls = calls["jit_decorate"]
|
||||
assert module.mhc_post_tilelang() == "mhc_post_tilelang"
|
||||
if current_platform.is_rocm():
|
||||
assert calls["jit_decorate"] == 1
|
||||
else:
|
||||
assert calls["jit_decorate"] == decorated_calls
|
||||
assert calls["compiled_call"] == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Test requires ROCm")
|
||||
def test_deepseek_v4_import_and_jit_monitor_do_not_hijack_hip_symbols(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
if importlib.util.find_spec("tilelang") is None:
|
||||
pytest.skip("Test requires TileLang to be installed")
|
||||
|
||||
class DlInfo(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("dli_fname", ctypes.c_char_p),
|
||||
("dli_fbase", ctypes.c_void_p),
|
||||
("dli_sname", ctypes.c_char_p),
|
||||
("dli_saddr", ctypes.c_void_p),
|
||||
]
|
||||
|
||||
libdl = ctypes.CDLL("libdl.so.2")
|
||||
dlsym = libdl.dlsym
|
||||
dlsym.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
|
||||
dlsym.restype = ctypes.c_void_p
|
||||
dladdr = libdl.dladdr
|
||||
dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(DlInfo)]
|
||||
dladdr.restype = ctypes.c_int
|
||||
|
||||
from vllm.model_executor.layers import mhc # noqa: F401
|
||||
from vllm.models import deepseek_v4 # noqa: F401
|
||||
from vllm.utils import jit_monitor
|
||||
|
||||
monkeypatch.setattr(jit_monitor, "_active", False)
|
||||
monkeypatch.setattr(jit_monitor, "_setup_triton_autotuning_print", lambda: None)
|
||||
monkeypatch.setattr(jit_monitor, "_setup_triton_jit_hook", lambda: None)
|
||||
monkeypatch.setattr(jit_monitor, "_setup_cutedsl_jit_hook", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
jit_monitor,
|
||||
"_setup_tilelang_jit_hook",
|
||||
lambda: pytest.fail("TileLang JIT monitor must not run on ROCm"),
|
||||
)
|
||||
jit_monitor.activate()
|
||||
|
||||
assert not any(
|
||||
name == "tilelang" or name.startswith("tilelang.") for name in sys.modules
|
||||
)
|
||||
|
||||
address = dlsym(None, b"hipFree")
|
||||
assert address is not None, "hipFree is not available in the global symbol table"
|
||||
info = DlInfo()
|
||||
assert dladdr(address, ctypes.byref(info))
|
||||
source = info.dli_fname.decode() if info.dli_fname else "<unknown>"
|
||||
assert "libhip_stub.so" not in source, source
|
||||
assert "libamdhip64.so" in source, source
|
||||
@@ -92,6 +92,15 @@ CHECK_IMPORTS = {
|
||||
),
|
||||
allowed_files={"vllm/triton_utils/importing.py"},
|
||||
),
|
||||
"tilelang": ForbiddenImport(
|
||||
pattern=r"^(from|import)\s+tilelang(\s|\.|$)",
|
||||
tip="Use 'from vllm.tilelang_utils import tilelang, T' instead.",
|
||||
allowed_pattern=re.compile(
|
||||
r"from\s+vllm\.tilelang_utils\s+import\s+"
|
||||
r"(tilelang|T|T, tilelang|tilelang, T)\b"
|
||||
),
|
||||
allowed_files={"vllm/tilelang_utils/__init__.py"},
|
||||
),
|
||||
"huggingface_hub repo API": ForbiddenImport(
|
||||
# Catch `from huggingface_hub import <fn>`, including parenthesized,
|
||||
# multi-line imports.
|
||||
@@ -182,6 +191,29 @@ def test_regex():
|
||||
f"(expected {should_match}, got {result})"
|
||||
)
|
||||
|
||||
tilelang_cases = [
|
||||
# Should match
|
||||
("import tilelang", True),
|
||||
("import tilelang.language as T", True),
|
||||
("from tilelang.jit import JITImpl", True),
|
||||
("from tilelang.jit.kernel import JITKernel", True),
|
||||
# Should not match: indented (local) imports are allowed, mirroring
|
||||
# the "triton" rule, so mocked-module test imports are not flagged.
|
||||
(" import tilelang", False),
|
||||
(" from tilelang.jit import JITImpl", False),
|
||||
("from vllm.tilelang_utils import tilelang", False),
|
||||
("from vllm.tilelang_utils import T", False),
|
||||
("from vllm.tilelang_utils import T, tilelang", False),
|
||||
("from vllm.tilelang_utils import tilelang, T", False),
|
||||
("import tilelang_kernels", False),
|
||||
]
|
||||
for i, (content, should_match) in enumerate(tilelang_cases):
|
||||
result = matches("tilelang", content)
|
||||
assert result == should_match, (
|
||||
f"tilelang case {i} failed: {content!r} "
|
||||
f"(expected {should_match}, got {result})"
|
||||
)
|
||||
|
||||
hf_cases = [
|
||||
# Should match
|
||||
("from huggingface_hub import snapshot_download", True),
|
||||
|
||||
@@ -1,29 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from functools import cache
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_tilelang
|
||||
from vllm.tilelang_utils import T, tilelang, tilelang_jit
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
# TileLang is used for MHC on CUDA and ROCm. Keep non-GPU imports cheap so
|
||||
# registering the Python wrapper modules does not require TileLang everywhere.
|
||||
if TYPE_CHECKING or current_platform.is_cuda_alike():
|
||||
if not has_tilelang():
|
||||
raise ImportError(
|
||||
"tilelang is required for mhc but is not installed. Install it with "
|
||||
"`pip install tilelang`."
|
||||
)
|
||||
import tilelang
|
||||
import tilelang.language as T
|
||||
else:
|
||||
tilelang = None # type: ignore[assignment]
|
||||
T = None # type: ignore[assignment]
|
||||
|
||||
ENABLE_PDL = current_platform.is_arch_support_pdl() and current_platform.is_cuda()
|
||||
|
||||
|
||||
@@ -40,18 +28,7 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int:
|
||||
return split_k
|
||||
|
||||
|
||||
pass_configs: dict[tilelang.PassConfigKey, Any] = {
|
||||
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
|
||||
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
|
||||
}
|
||||
|
||||
if current_platform.is_cuda():
|
||||
pass_configs[tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL] = 10
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs=pass_configs,
|
||||
)
|
||||
@tilelang_jit
|
||||
def mhc_pre_big_fuse_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
@@ -191,9 +168,7 @@ def mhc_pre_big_fuse_tilelang(
|
||||
# Copied from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/layers/mhc.py#L478
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs=pass_configs,
|
||||
)
|
||||
@tilelang_jit
|
||||
def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
@@ -353,9 +328,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs=pass_configs,
|
||||
)
|
||||
@tilelang_jit
|
||||
def mhc_pre_big_fuse_broadcast_with_norm_tilelang(
|
||||
gemm_out_mul,
|
||||
gemm_out_sqrsum,
|
||||
@@ -517,9 +490,7 @@ def mhc_pre_big_fuse_broadcast_with_norm_tilelang(
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs=pass_configs,
|
||||
)
|
||||
@tilelang_jit
|
||||
def mhc_fused_tilelang(
|
||||
comb_mix,
|
||||
residual_in,
|
||||
@@ -640,9 +611,7 @@ def mhc_fused_tilelang(
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs=pass_configs,
|
||||
)
|
||||
@tilelang_jit
|
||||
def mhc_post_tilelang(
|
||||
a,
|
||||
b,
|
||||
@@ -695,9 +664,7 @@ def mhc_post_tilelang(
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs=pass_configs,
|
||||
)
|
||||
@tilelang_jit
|
||||
def hc_prenorm_gemm_tilelang(
|
||||
x,
|
||||
fn,
|
||||
@@ -781,9 +748,7 @@ def hc_prenorm_gemm_tilelang(
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs=pass_configs,
|
||||
)
|
||||
@tilelang_jit
|
||||
def hc_prenorm_gemm_block_m_tilelang(
|
||||
x,
|
||||
fn,
|
||||
@@ -878,9 +843,7 @@ def hc_prenorm_gemm_block_m_tilelang(
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
@tilelang.jit(
|
||||
pass_configs=pass_configs,
|
||||
)
|
||||
@tilelang_jit
|
||||
def hc_head_fuse_tilelang(
|
||||
residual,
|
||||
fn,
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
from functools import cache
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_tilelang
|
||||
|
||||
if TYPE_CHECKING or current_platform.is_cuda():
|
||||
if not has_tilelang():
|
||||
raise ImportError(
|
||||
"tilelang is required for mhc but is not installed. Install it with "
|
||||
"`pip install tilelang`."
|
||||
)
|
||||
import tilelang
|
||||
import tilelang.language as T
|
||||
else:
|
||||
tilelang = None # type: ignore[assignment]
|
||||
T = None # type: ignore[assignment]
|
||||
|
||||
|
||||
def _ensure_tilelang_imported() -> None:
|
||||
"""Bind the `tilelang` and `T` module globals, importing them if needed.
|
||||
|
||||
On ROCm, this runs on the first kernel call instead of at import time.
|
||||
|
||||
Raises:
|
||||
ImportError: If TileLang is not installed.
|
||||
"""
|
||||
global T, tilelang
|
||||
|
||||
if tilelang is not None:
|
||||
return
|
||||
if not has_tilelang():
|
||||
raise ImportError(
|
||||
"tilelang is required for mhc but is not installed. Install it with "
|
||||
"`pip install tilelang`."
|
||||
)
|
||||
import tilelang as tilelang_module
|
||||
import tilelang.language as tilelang_language
|
||||
|
||||
tilelang = tilelang_module
|
||||
T = tilelang_language
|
||||
|
||||
|
||||
@cache
|
||||
def _get_pass_configs() -> dict[Any, Any]:
|
||||
_ensure_tilelang_imported()
|
||||
pass_configs: dict[Any, Any] = {
|
||||
tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True,
|
||||
tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True,
|
||||
}
|
||||
if current_platform.is_cuda():
|
||||
pass_configs[tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL] = 10
|
||||
return pass_configs
|
||||
|
||||
|
||||
def tilelang_jit(kernel_function: Callable[..., Any]) -> Callable[..., Any]:
|
||||
"""Apply `tilelang.jit`, deferring until first call on ROCm.
|
||||
|
||||
ROCm defers JIT decoration so importing the caller's module does not
|
||||
require TileLang immediately. CUDA keeps the eager decoration behavior.
|
||||
|
||||
The kernel body parsed by TileLang references `T` as an unqualified
|
||||
global, so on the deferred ROCm path this rebinds `T`/`tilelang` in the
|
||||
decorated function's own module globals once they become available.
|
||||
"""
|
||||
if not current_platform.is_rocm():
|
||||
_ensure_tilelang_imported()
|
||||
return tilelang.jit(pass_configs=_get_pass_configs())(kernel_function)
|
||||
|
||||
compiled_kernel: Callable[..., Any] | None = None
|
||||
|
||||
@functools.wraps(kernel_function)
|
||||
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
nonlocal compiled_kernel
|
||||
if compiled_kernel is None:
|
||||
_ensure_tilelang_imported()
|
||||
kernel_function.__globals__["tilelang"] = tilelang
|
||||
kernel_function.__globals__["T"] = T
|
||||
compiled_kernel = tilelang.jit(pass_configs=_get_pass_configs())(
|
||||
kernel_function
|
||||
)
|
||||
return compiled_kernel(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
@@ -412,6 +412,21 @@ def _has_module(module_name: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
@cache
|
||||
def _has_module_spec(module_name: str) -> bool:
|
||||
"""Return True if *module_name* is installed, without importing it.
|
||||
|
||||
Unlike [`_has_module`][vllm.utils.import_utils._has_module], this only
|
||||
resolves the import spec. It therefore does not pay the import cost of
|
||||
heavyweight modules, at the price of not verifying that native
|
||||
dependencies (shared libraries, etc.) are satisfied. The result is cached.
|
||||
"""
|
||||
try:
|
||||
return importlib.util.find_spec(module_name) is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def has_deep_ep() -> bool:
|
||||
"""Whether the optional `deep_ep` package is available."""
|
||||
return _has_module("deep_ep")
|
||||
@@ -504,8 +519,13 @@ def has_triton_kernels() -> bool:
|
||||
|
||||
@cache
|
||||
def has_tilelang() -> bool:
|
||||
"""Whether the optional `tilelang` package is available."""
|
||||
if not _has_module("tilelang"):
|
||||
"""Whether the optional `tilelang` package is available.
|
||||
|
||||
Only the import spec is checked: importing `tilelang` is expensive, so
|
||||
callers must import it lazily at their point of use rather than relying
|
||||
on this function to have imported it already.
|
||||
"""
|
||||
if not _has_module_spec("tilelang"):
|
||||
return False
|
||||
# ROCm-only guard, imported lazily to avoid loading rocm on CUDA.
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -30,6 +30,7 @@ from contextlib import suppress
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils.importing import HAS_TRITON
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -74,7 +75,12 @@ def activate(*, mode: JitMonitorMode = "warn", verbose: bool = False) -> None:
|
||||
_setup_triton_autotuning_print()
|
||||
_setup_triton_jit_hook()
|
||||
_setup_cutedsl_jit_hook()
|
||||
_setup_tilelang_jit_hook()
|
||||
|
||||
# Refer to #51159. tilelang ships broken symbols
|
||||
# on rocm.
|
||||
# TODO: Remove the guard once tilelang upstream is fixed.
|
||||
if not current_platform.is_rocm():
|
||||
_setup_tilelang_jit_hook()
|
||||
|
||||
logger.info(
|
||||
"Kernel JIT monitor activated; monitored JIT compilations during "
|
||||
|
||||
Reference in New Issue
Block a user