[Feat] Add runtime monitor for post-warmup TileLang compilation (#46718)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import inspect
|
||||
import os
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
@@ -19,11 +20,15 @@ def _reset_monitor():
|
||||
jit_monitor._mode = "warn"
|
||||
jit_monitor._verbose = False
|
||||
jit_monitor._cutedsl_hook_installed = False
|
||||
jit_monitor._tilelang_hook_installed = False
|
||||
jit_monitor._tilelang_jitimpl_compile_depth = 0
|
||||
yield
|
||||
jit_monitor._active = False
|
||||
jit_monitor._mode = "warn"
|
||||
jit_monitor._verbose = False
|
||||
jit_monitor._cutedsl_hook_installed = False
|
||||
jit_monitor._tilelang_hook_installed = False
|
||||
jit_monitor._tilelang_jitimpl_compile_depth = 0
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -55,6 +60,45 @@ def _fake_cute_compile(*args, **kwargs):
|
||||
return "compiled"
|
||||
|
||||
|
||||
def _fake_tilelang_import_modules():
|
||||
"""Fake Python's TileLang modules touched by ``jit_monitor.activate``."""
|
||||
|
||||
class FakeJITKernel:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
class FakeJITImpl:
|
||||
def __init__(self, func, signature):
|
||||
self.func = func
|
||||
self.signature = signature
|
||||
self.mode = "lazy"
|
||||
self._kernel_cache = {}
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
key, _ = self.func.parse_args(*args, **kwargs)
|
||||
kernel = self._kernel_cache.get(key)
|
||||
if kernel is None:
|
||||
kernel = "compiled"
|
||||
self._kernel_cache[key] = kernel
|
||||
return kernel
|
||||
|
||||
fake_kernel = cast(Any, ModuleType("tilelang.jit.kernel"))
|
||||
fake_kernel.JITKernel = FakeJITKernel
|
||||
|
||||
fake_jit = cast(Any, ModuleType("tilelang.jit"))
|
||||
fake_jit.JITImpl = FakeJITImpl
|
||||
fake_jit.kernel = fake_kernel
|
||||
|
||||
fake_tilelang = cast(Any, ModuleType("tilelang"))
|
||||
fake_tilelang.jit = fake_jit
|
||||
|
||||
return {
|
||||
"tilelang": fake_tilelang,
|
||||
"tilelang.jit": fake_jit,
|
||||
"tilelang.jit.kernel": fake_kernel,
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_jit_modules(fake_knobs, *, cute_compile=_fake_cute_compile):
|
||||
"""Patch the Triton and CuTeDSL imports touched by ``jit_monitor.activate``."""
|
||||
@@ -66,6 +110,7 @@ def _patch_jit_modules(fake_knobs, *, cute_compile=_fake_cute_compile):
|
||||
{
|
||||
"triton": fake_triton,
|
||||
**_fake_cute_import_modules(cute_compile),
|
||||
**_fake_tilelang_import_modules(),
|
||||
},
|
||||
),
|
||||
mock.patch.object(jit_monitor, "HAS_TRITON", True),
|
||||
@@ -276,6 +321,116 @@ class TestCuTeDSLHook:
|
||||
cute.compile(lambda: None, "arg", option=True)
|
||||
|
||||
|
||||
class TestTileLangHook:
|
||||
def test_jit_kernel_logs_warning(self):
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
from tilelang.jit.kernel import JITKernel
|
||||
|
||||
func = SimpleNamespace(attrs={"global_symbol": "tl_kernel"})
|
||||
jit_monitor.activate()
|
||||
with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once:
|
||||
JITKernel(func=func, out_idx=None, execution_backend="tvm_ffi")
|
||||
|
||||
warning_once.assert_called_once()
|
||||
msg = warning_once.call_args[0][0] % warning_once.call_args[0][1:]
|
||||
assert "TileLang JIT compilation during inference" in msg
|
||||
assert "tl_kernel" in msg
|
||||
|
||||
def test_jit_impl_logs_warning(self):
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
from tilelang.jit import JITImpl
|
||||
|
||||
def tilelang_fn(
|
||||
gemm_out_mul,
|
||||
hidden_size: int,
|
||||
n_splits: int = 1,
|
||||
hc_mult: int = 4,
|
||||
):
|
||||
return None
|
||||
|
||||
class FakeFunc:
|
||||
orig_func = tilelang_fn
|
||||
|
||||
def parse_args(self, *args, **kwargs):
|
||||
return (
|
||||
(
|
||||
"tilelang_key",
|
||||
kwargs["hidden_size"],
|
||||
kwargs.get("n_splits", 1),
|
||||
),
|
||||
{},
|
||||
)
|
||||
|
||||
def set_mode(self, mode):
|
||||
self.mode = mode
|
||||
|
||||
tensor = SimpleNamespace(
|
||||
shape=(2, 16, 24),
|
||||
dtype="float32",
|
||||
device="cuda:0",
|
||||
)
|
||||
impl = JITImpl(FakeFunc(), inspect.signature(tilelang_fn))
|
||||
|
||||
jit_monitor.activate()
|
||||
with (
|
||||
mock.patch.object(jit_monitor.logger, "warning_once") as warning_once,
|
||||
mock.patch.object(jit_monitor.logger, "warning") as warning,
|
||||
):
|
||||
impl(tensor, hidden_size=7168, n_splits=2)
|
||||
|
||||
warning_once.assert_called_once()
|
||||
warning.assert_not_called()
|
||||
msg = warning_once.call_args[0][0] % warning_once.call_args[0][1:]
|
||||
assert "TileLang JIT compilation during inference" in msg
|
||||
assert "tilelang_fn" in msg
|
||||
|
||||
def test_jit_impl_does_not_log_on_cache_hit(self):
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
from tilelang.jit import JITImpl
|
||||
|
||||
def tilelang_fn(gemm_out_mul, n_splits: int = 1):
|
||||
return None
|
||||
|
||||
class FakeFunc:
|
||||
orig_func = tilelang_fn
|
||||
|
||||
def parse_args(self, *args, **kwargs):
|
||||
return (("tilelang_key", kwargs.get("n_splits", 1)), {})
|
||||
|
||||
def set_mode(self, mode):
|
||||
self.mode = mode
|
||||
|
||||
tensor = SimpleNamespace(shape=(2, 16, 24), dtype="float32")
|
||||
impl = JITImpl(FakeFunc(), inspect.signature(tilelang_fn))
|
||||
|
||||
jit_monitor.activate()
|
||||
with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once:
|
||||
impl(tensor, n_splits=2)
|
||||
impl(tensor, n_splits=2)
|
||||
|
||||
warning_once.assert_called_once()
|
||||
|
||||
def test_from_database_does_not_log(self):
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
from tilelang.jit.kernel import JITKernel
|
||||
|
||||
func = SimpleNamespace(attrs={"global_symbol": "cached_tl_kernel"})
|
||||
jit_monitor.activate()
|
||||
with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once:
|
||||
JITKernel(func=func, from_database=True)
|
||||
|
||||
warning_once.assert_not_called()
|
||||
|
||||
def test_error_mode_raises(self):
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
from tilelang.jit.kernel import JITKernel
|
||||
|
||||
func = SimpleNamespace(attrs={"global_symbol": "error_tl_kernel"})
|
||||
jit_monitor.activate(mode="error")
|
||||
with pytest.raises(RuntimeError, match="TileLang JIT compilation"):
|
||||
JITKernel(func=func)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Integration tests (real Triton + GPU)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
+212
-1
@@ -18,13 +18,16 @@ Currently monitors:
|
||||
- Triton ``@triton.autotune`` cache misses (via ``knobs.autotuning.print``)
|
||||
- Triton ``@triton.jit`` first-time compilations
|
||||
(via ``knobs.runtime.jit_post_compile_hook``)
|
||||
- TileLang ``@tilelang.jit`` first-time compilations
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import importlib
|
||||
import os
|
||||
from collections.abc import Iterator, Mapping
|
||||
from typing import Literal
|
||||
from contextlib import suppress
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils.importing import HAS_TRITON
|
||||
@@ -36,6 +39,8 @@ _active: bool = False
|
||||
_mode: JitMonitorMode = "warn"
|
||||
_verbose: bool = False
|
||||
_cutedsl_hook_installed: bool = False
|
||||
_tilelang_hook_installed: bool = False
|
||||
_tilelang_jitimpl_compile_depth: int = 0
|
||||
|
||||
|
||||
def is_active() -> bool:
|
||||
@@ -69,6 +74,7 @@ def activate(*, mode: JitMonitorMode = "warn", verbose: bool = False) -> None:
|
||||
_setup_triton_autotuning_print()
|
||||
_setup_triton_jit_hook()
|
||||
_setup_cutedsl_jit_hook()
|
||||
_setup_tilelang_jit_hook()
|
||||
|
||||
logger.info(
|
||||
"Kernel JIT monitor activated; monitored JIT compilations during "
|
||||
@@ -290,6 +296,211 @@ def _setup_cutedsl_jit_hook() -> None:
|
||||
_cutedsl_hook_installed = True
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# TileLang JIT compilation hook
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _tilelang_arg(
|
||||
args: tuple[object, ...],
|
||||
kwargs: Mapping[str, object],
|
||||
index: int,
|
||||
name: str,
|
||||
default: object = None,
|
||||
) -> object:
|
||||
if len(args) > index:
|
||||
return args[index]
|
||||
return kwargs.get(name, default)
|
||||
|
||||
|
||||
def _tilelang_kernel_name(func: object) -> str:
|
||||
attrs = getattr(func, "attrs", None)
|
||||
global_symbol = None
|
||||
get_attr = getattr(attrs, "get", None)
|
||||
if callable(get_attr):
|
||||
try:
|
||||
global_symbol = get_attr("global_symbol")
|
||||
except Exception:
|
||||
global_symbol = None
|
||||
elif isinstance(attrs, Mapping):
|
||||
global_symbol = attrs.get("global_symbol")
|
||||
|
||||
if global_symbol is not None:
|
||||
return str(global_symbol)
|
||||
|
||||
name = getattr(func, "__name__", None)
|
||||
if name is not None:
|
||||
return str(name)
|
||||
return func.__class__.__name__ if func is not None else "<unknown>"
|
||||
|
||||
|
||||
def _tilelang_call_kwargs(kwargs: Mapping[str, object]) -> dict[str, object]:
|
||||
call_kwargs = dict(kwargs)
|
||||
tune_params = call_kwargs.pop("__tune_params", {})
|
||||
if isinstance(tune_params, Mapping):
|
||||
call_kwargs.update(tune_params)
|
||||
return call_kwargs
|
||||
|
||||
|
||||
def _tilelang_cache_miss_key(
|
||||
jit_impl: object,
|
||||
args: tuple[object, ...],
|
||||
kwargs: Mapping[str, object],
|
||||
) -> object | None:
|
||||
if kwargs.get("__return_compile_arguments", False):
|
||||
return None
|
||||
|
||||
call_kwargs = _tilelang_call_kwargs(kwargs)
|
||||
func = getattr(jit_impl, "func", None)
|
||||
parse_args = getattr(func, "parse_args", None)
|
||||
if not callable(parse_args):
|
||||
return None
|
||||
|
||||
try:
|
||||
if getattr(jit_impl, "mode", None) == "auto":
|
||||
impl = cast(Any, jit_impl)
|
||||
mode = impl._infer_jit_mode(*args, **call_kwargs)
|
||||
impl.mode = mode
|
||||
if func is not None:
|
||||
func.set_mode(mode)
|
||||
key, _ = parse_args(*args, **call_kwargs)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
cache = getattr(jit_impl, "_kernel_cache", {})
|
||||
if isinstance(cache, Mapping):
|
||||
return key if key not in cache else None
|
||||
|
||||
get = getattr(cache, "get", None)
|
||||
if not callable(get):
|
||||
return key
|
||||
try:
|
||||
return key if get(key) is None else None
|
||||
except Exception:
|
||||
return key
|
||||
|
||||
|
||||
def _format_tilelang_runtime_shapes(
|
||||
jit_impl: object,
|
||||
args: tuple[object, ...],
|
||||
kwargs: Mapping[str, object],
|
||||
) -> str:
|
||||
signature = getattr(jit_impl, "signature", None)
|
||||
bind_partial = getattr(signature, "bind_partial", None)
|
||||
if not callable(bind_partial):
|
||||
return "{}"
|
||||
|
||||
try:
|
||||
bound = bind_partial(*args, **_tilelang_call_kwargs(kwargs))
|
||||
bound.apply_defaults()
|
||||
except Exception:
|
||||
return "{}"
|
||||
|
||||
items = []
|
||||
for name, value in bound.arguments.items():
|
||||
if str(name).startswith("__"):
|
||||
continue
|
||||
if not (hasattr(value, "shape") and hasattr(value, "dtype")):
|
||||
continue
|
||||
shape = getattr(value, "shape", None)
|
||||
if shape is not None:
|
||||
with suppress(TypeError):
|
||||
shape = tuple(shape)
|
||||
items.append(f"{name}={_safe_repr(shape, max_len=80)}")
|
||||
return "{" + ", ".join(items) + "}"
|
||||
|
||||
|
||||
def _format_verbose_tilelang_compile_details(
|
||||
jit_impl: object,
|
||||
args: tuple[object, ...],
|
||||
kwargs: Mapping[str, object],
|
||||
cache_key: object,
|
||||
) -> str:
|
||||
return (
|
||||
f"cache_key={_safe_repr(cache_key, max_len=400)}; "
|
||||
f"runtime_shapes={_format_tilelang_runtime_shapes(jit_impl, args, kwargs)}"
|
||||
)
|
||||
|
||||
|
||||
def _log_tilelang_jit_compile(
|
||||
fn_name: str,
|
||||
detail: str | None = None,
|
||||
) -> None:
|
||||
_handle_jit_event(
|
||||
backend="TileLang",
|
||||
event="JIT compilation",
|
||||
fn_name=fn_name,
|
||||
detail=detail,
|
||||
)
|
||||
|
||||
|
||||
def _setup_tilelang_jit_hook() -> None:
|
||||
"""Wrap TileLang JIT entry points to warn on compilation."""
|
||||
global _tilelang_hook_installed
|
||||
if _tilelang_hook_installed:
|
||||
return
|
||||
|
||||
try:
|
||||
tilelang_kernel = importlib.import_module("tilelang.jit.kernel")
|
||||
except Exception:
|
||||
logger.debug("TileLang is not available; skipping TileLang JIT monitor.")
|
||||
return
|
||||
|
||||
jit_kernel_cls = getattr(tilelang_kernel, "JITKernel", None)
|
||||
if jit_kernel_cls is None:
|
||||
logger.debug(
|
||||
"TileLang JITKernel is unavailable; skipping TileLang JIT monitor."
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
tilelang_jit = importlib.import_module("tilelang.jit")
|
||||
except Exception:
|
||||
tilelang_jit = None
|
||||
jit_impl_cls = getattr(tilelang_jit, "JITImpl", None)
|
||||
original_init = jit_kernel_cls.__init__
|
||||
|
||||
@functools.wraps(original_init)
|
||||
def _init_with_monitor(self, *args, **kwargs):
|
||||
from_database = bool(_tilelang_arg(args, kwargs, 7, "from_database", False))
|
||||
if not from_database and _tilelang_jitimpl_compile_depth == 0:
|
||||
func = _tilelang_arg(args, kwargs, 0, "func")
|
||||
_log_tilelang_jit_compile(_tilelang_kernel_name(func))
|
||||
return original_init(self, *args, **kwargs)
|
||||
|
||||
jit_kernel_cls.__init__ = _init_with_monitor
|
||||
|
||||
if jit_impl_cls is not None:
|
||||
original_call = jit_impl_cls.__call__
|
||||
|
||||
@functools.wraps(original_call)
|
||||
def _call_with_monitor(self, *args, **kwargs):
|
||||
global _tilelang_jitimpl_compile_depth
|
||||
cache_key = _tilelang_cache_miss_key(self, args, kwargs)
|
||||
if cache_key is None:
|
||||
return original_call(self, *args, **kwargs)
|
||||
|
||||
_tilelang_jitimpl_compile_depth += 1
|
||||
try:
|
||||
detail = None
|
||||
if _verbose:
|
||||
detail = _format_verbose_tilelang_compile_details(
|
||||
self, args, kwargs, cache_key
|
||||
)
|
||||
func = getattr(self, "func", None)
|
||||
orig_func = getattr(func, "orig_func", None)
|
||||
_log_tilelang_jit_compile(
|
||||
_tilelang_kernel_name(orig_func or func), detail
|
||||
)
|
||||
return original_call(self, *args, **kwargs)
|
||||
finally:
|
||||
_tilelang_jitimpl_compile_depth -= 1
|
||||
|
||||
jit_impl_cls.__call__ = _call_with_monitor
|
||||
|
||||
_tilelang_hook_installed = True
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def numba_workqueue_threading_layer() -> Iterator[None]:
|
||||
"""Force numba's fork-safe `workqueue` threading layer for this block.
|
||||
|
||||
Reference in New Issue
Block a user