[Feat] Add runtime monitor for post-warmup CuTeDSL compilation (#46167)
This commit is contained in:
@@ -214,6 +214,17 @@ def test_jit_monitor_verbose_arg():
|
||||
assert EngineArgs(model="test", jit_monitor_verbose=True).jit_monitor_verbose
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["warn", "error"])
|
||||
def test_jit_monitor_mode_arg(mode):
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
args = parser.parse_args(["--jit-monitor-mode", mode])
|
||||
|
||||
assert args.jit_monitor_mode == mode
|
||||
engine_args = EngineArgs(model="test", jit_monitor_mode=mode)
|
||||
assert engine_args.jit_monitor_mode == mode
|
||||
assert engine_args.create_observability_config().jit_monitor_mode == mode
|
||||
|
||||
|
||||
def test_hf_token_get_kwargs():
|
||||
kwargs = get_kwargs(ModelConfig)["hf_token"]
|
||||
|
||||
|
||||
+113
-18
@@ -3,26 +3,31 @@
|
||||
import os
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.triton_utils import jit_monitor
|
||||
from vllm.utils import jit_monitor
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_monitor():
|
||||
"""Reset global monitor state between tests."""
|
||||
jit_monitor._active = False
|
||||
jit_monitor._mode = "warn"
|
||||
jit_monitor._verbose = False
|
||||
jit_monitor._cutedsl_hook_installed = False
|
||||
yield
|
||||
jit_monitor._active = False
|
||||
jit_monitor._mode = "warn"
|
||||
jit_monitor._verbose = False
|
||||
jit_monitor._cutedsl_hook_installed = False
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers — lightweight stand-ins for triton.knobs
|
||||
# Helpers — lightweight stand-ins for the modules ``activate()`` patches
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -33,12 +38,36 @@ def _make_fake_knobs(*, autotuning_print=False, jit_hook=None):
|
||||
return SimpleNamespace(autotuning=autotuning, runtime=runtime)
|
||||
|
||||
|
||||
def _fake_cute_import_modules(compile_fn):
|
||||
"""Fake Python's parent package + submodule for ``import cutlass.cute``."""
|
||||
fake_cute = cast(Any, ModuleType("cutlass.cute"))
|
||||
fake_cute.compile = compile_fn
|
||||
fake_parent_package = cast(Any, ModuleType("cutlass"))
|
||||
fake_parent_package.__path__ = []
|
||||
fake_parent_package.cute = fake_cute
|
||||
return {
|
||||
"cutlass": fake_parent_package,
|
||||
"cutlass.cute": fake_cute,
|
||||
}
|
||||
|
||||
|
||||
def _fake_cute_compile(*args, **kwargs):
|
||||
return "compiled"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_triton_knobs(fake_knobs):
|
||||
"""Context manager that makes ``from triton import knobs`` return *fake_knobs*."""
|
||||
fake_triton = SimpleNamespace(knobs=fake_knobs)
|
||||
def _patch_jit_modules(fake_knobs, *, cute_compile=_fake_cute_compile):
|
||||
"""Patch the Triton and CuTeDSL imports touched by ``jit_monitor.activate``."""
|
||||
fake_triton = cast(Any, ModuleType("triton"))
|
||||
fake_triton.knobs = fake_knobs
|
||||
with (
|
||||
mock.patch.dict(sys.modules, {"triton": fake_triton}),
|
||||
mock.patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"triton": fake_triton,
|
||||
**_fake_cute_import_modules(cute_compile),
|
||||
},
|
||||
),
|
||||
mock.patch.object(jit_monitor, "HAS_TRITON", True),
|
||||
):
|
||||
yield
|
||||
@@ -52,13 +81,13 @@ def _patch_triton_knobs(fake_knobs):
|
||||
class TestActivateBasic:
|
||||
def test_sets_active(self):
|
||||
assert not jit_monitor.is_active()
|
||||
with _patch_triton_knobs(_make_fake_knobs()):
|
||||
with _patch_jit_modules(_make_fake_knobs()):
|
||||
jit_monitor.activate()
|
||||
assert jit_monitor.is_active()
|
||||
|
||||
def test_idempotent(self):
|
||||
fake = _make_fake_knobs()
|
||||
with _patch_triton_knobs(fake):
|
||||
with _patch_jit_modules(fake):
|
||||
jit_monitor.activate()
|
||||
first_hook = fake.runtime.jit_post_compile_hook
|
||||
jit_monitor.activate()
|
||||
@@ -67,17 +96,21 @@ class TestActivateBasic:
|
||||
def test_logs_info_on_activation(self):
|
||||
with (
|
||||
mock.patch.object(jit_monitor.logger, "info") as m,
|
||||
_patch_triton_knobs(_make_fake_knobs()),
|
||||
_patch_jit_modules(_make_fake_knobs()),
|
||||
):
|
||||
jit_monitor.activate()
|
||||
m.assert_called_once()
|
||||
assert "Kernel JIT monitor activated" in m.call_args[0][0]
|
||||
|
||||
def test_rejects_unknown_mode(self):
|
||||
with pytest.raises(ValueError, match="Unsupported JIT monitor mode"):
|
||||
jit_monitor.activate(mode="panic") # type: ignore[arg-type]
|
||||
|
||||
|
||||
class TestAutotuningPrint:
|
||||
def test_enables_autotuning_print(self):
|
||||
fake = _make_fake_knobs(autotuning_print=False)
|
||||
with _patch_triton_knobs(fake):
|
||||
with _patch_jit_modules(fake):
|
||||
jit_monitor.activate()
|
||||
assert fake.autotuning.print is True
|
||||
|
||||
@@ -85,7 +118,7 @@ class TestAutotuningPrint:
|
||||
fake = _make_fake_knobs(autotuning_print=False)
|
||||
with (
|
||||
mock.patch.dict(os.environ, {"TRITON_PRINT_AUTOTUNING": "0"}),
|
||||
_patch_triton_knobs(fake),
|
||||
_patch_jit_modules(fake),
|
||||
):
|
||||
jit_monitor.activate()
|
||||
assert fake.autotuning.print is False
|
||||
@@ -94,23 +127,23 @@ class TestAutotuningPrint:
|
||||
fake = _make_fake_knobs(autotuning_print=True)
|
||||
with (
|
||||
mock.patch.dict(os.environ, {"TRITON_PRINT_AUTOTUNING": "1"}),
|
||||
_patch_triton_knobs(fake),
|
||||
_patch_jit_modules(fake),
|
||||
):
|
||||
jit_monitor.activate()
|
||||
assert fake.autotuning.print is True
|
||||
|
||||
|
||||
class TestJitHook:
|
||||
class TestTritonJitHook:
|
||||
def test_hook_registered(self):
|
||||
fake = _make_fake_knobs()
|
||||
assert fake.runtime.jit_post_compile_hook is None
|
||||
with _patch_triton_knobs(fake):
|
||||
with _patch_jit_modules(fake):
|
||||
jit_monitor.activate()
|
||||
assert fake.runtime.jit_post_compile_hook is not None
|
||||
|
||||
def test_hook_logs_warning(self):
|
||||
fake = _make_fake_knobs()
|
||||
with _patch_triton_knobs(fake):
|
||||
with _patch_jit_modules(fake):
|
||||
jit_monitor.activate()
|
||||
|
||||
hook = fake.runtime.jit_post_compile_hook
|
||||
@@ -138,7 +171,7 @@ class TestJitHook:
|
||||
def test_hook_chains_existing_hook(self):
|
||||
existing = mock.MagicMock(return_value="existing_result")
|
||||
fake = _make_fake_knobs(jit_hook=existing)
|
||||
with _patch_triton_knobs(fake):
|
||||
with _patch_jit_modules(fake):
|
||||
jit_monitor.activate()
|
||||
|
||||
hook = fake.runtime.jit_post_compile_hook
|
||||
@@ -158,7 +191,7 @@ class TestJitHook:
|
||||
|
||||
def test_hook_works_without_existing_hook(self):
|
||||
fake = _make_fake_knobs(jit_hook=None)
|
||||
with _patch_triton_knobs(fake):
|
||||
with _patch_jit_modules(fake):
|
||||
jit_monitor.activate()
|
||||
|
||||
hook = fake.runtime.jit_post_compile_hook
|
||||
@@ -173,6 +206,23 @@ class TestJitHook:
|
||||
)
|
||||
assert result is None
|
||||
|
||||
def test_error_mode_raises(self):
|
||||
fake = _make_fake_knobs()
|
||||
with _patch_jit_modules(fake):
|
||||
jit_monitor.activate(mode="error")
|
||||
|
||||
hook = fake.runtime.jit_post_compile_hook
|
||||
mock_fn = SimpleNamespace(name="error_kernel")
|
||||
with pytest.raises(RuntimeError, match="Triton kernel JIT compilation"):
|
||||
hook(
|
||||
key="k",
|
||||
repr="r",
|
||||
fn=mock_fn,
|
||||
compile=lambda: None,
|
||||
is_manual_warmup=False,
|
||||
already_compiled=False,
|
||||
)
|
||||
|
||||
|
||||
class TestNoTritonFallback:
|
||||
def test_activate_without_triton(self):
|
||||
@@ -181,6 +231,51 @@ class TestNoTritonFallback:
|
||||
assert jit_monitor.is_active()
|
||||
|
||||
|
||||
class TestCuTeDSLHook:
|
||||
def test_compile_logs_warning(self):
|
||||
def compile_fn(*args, **kwargs):
|
||||
return "compiled"
|
||||
|
||||
with _patch_jit_modules(_make_fake_knobs(), cute_compile=compile_fn):
|
||||
import cutlass.cute as cute
|
||||
|
||||
jit_monitor.activate()
|
||||
with mock.patch.object(jit_monitor.logger, "warning_once") as warning_once:
|
||||
result = cute.compile(lambda: None, "arg", option=True)
|
||||
|
||||
assert result == "compiled"
|
||||
warning_once.assert_called_once()
|
||||
msg = warning_once.call_args[0][0] % warning_once.call_args[0][1:]
|
||||
assert "CuTeDSL JIT compilation during inference" in msg
|
||||
|
||||
def test_compile_logs_verbose_warning(self):
|
||||
def compile_fn(*args, **kwargs):
|
||||
return "compiled"
|
||||
|
||||
with _patch_jit_modules(_make_fake_knobs(), cute_compile=compile_fn):
|
||||
import cutlass.cute as cute
|
||||
|
||||
jit_monitor.activate(verbose=True)
|
||||
with mock.patch.object(jit_monitor.logger, "warning") as warning:
|
||||
result = cute.compile(lambda: None, "arg", option=True)
|
||||
|
||||
assert result == "compiled"
|
||||
warning.assert_called_once()
|
||||
msg = warning.call_args[0][0] % warning.call_args[0][1:]
|
||||
assert "CuTeDSL JIT compilation during inference" in msg
|
||||
|
||||
def test_error_mode_raises(self):
|
||||
def compile_fn(*args, **kwargs):
|
||||
return "compiled"
|
||||
|
||||
with _patch_jit_modules(_make_fake_knobs(), cute_compile=compile_fn):
|
||||
import cutlass.cute as cute
|
||||
|
||||
jit_monitor.activate(mode="error")
|
||||
with pytest.raises(RuntimeError, match="CuTeDSL JIT compilation"):
|
||||
cute.compile(lambda: None, "arg", option=True)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Integration tests (real Triton + GPU)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -76,8 +76,11 @@ class ObservabilityConfig:
|
||||
This includes number of context/generation requests and tokens
|
||||
and the elapsed cpu time for the iteration."""
|
||||
|
||||
jit_monitor_mode: Literal["warn", "error"] = "warn"
|
||||
"""How to handle post-warmup JIT compilation events."""
|
||||
|
||||
jit_monitor_verbose: bool = False
|
||||
"""Log every Triton JIT compile with its dispatch key. This can emit many
|
||||
"""Log every monitored JIT compile with runtime details. This can emit many
|
||||
logs and add overhead, so it is intended for debugging."""
|
||||
|
||||
@cached_property
|
||||
|
||||
+22
-13
@@ -643,6 +643,7 @@ class EngineArgs:
|
||||
enable_logging_iteration_details: bool = (
|
||||
ObservabilityConfig.enable_logging_iteration_details
|
||||
)
|
||||
jit_monitor_mode: Literal["warn", "error"] = ObservabilityConfig.jit_monitor_mode
|
||||
jit_monitor_verbose: bool = ObservabilityConfig.jit_monitor_verbose
|
||||
enable_mm_processor_stats: bool = ObservabilityConfig.enable_mm_processor_stats
|
||||
scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
|
||||
@@ -1378,6 +1379,10 @@ class EngineArgs:
|
||||
"--enable-logging-iteration-details",
|
||||
**observability_kwargs["enable_logging_iteration_details"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--jit-monitor-mode",
|
||||
**observability_kwargs["jit_monitor_mode"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--jit-monitor-verbose",
|
||||
**observability_kwargs["jit_monitor_verbose"],
|
||||
@@ -1781,6 +1786,22 @@ class EngineArgs:
|
||||
cfg = json.loads(cfg)
|
||||
return DiffusionConfig(**cfg)
|
||||
|
||||
def create_observability_config(self) -> ObservabilityConfig:
|
||||
return ObservabilityConfig(
|
||||
show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
|
||||
otlp_traces_endpoint=self.otlp_traces_endpoint,
|
||||
collect_detailed_traces=self.collect_detailed_traces,
|
||||
kv_cache_metrics=self.kv_cache_metrics,
|
||||
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
|
||||
cudagraph_metrics=self.cudagraph_metrics,
|
||||
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
|
||||
enable_mfu_metrics=self.enable_mfu_metrics,
|
||||
enable_mm_processor_stats=self.enable_mm_processor_stats,
|
||||
enable_logging_iteration_details=self.enable_logging_iteration_details,
|
||||
jit_monitor_mode=self.jit_monitor_mode,
|
||||
jit_monitor_verbose=self.jit_monitor_verbose,
|
||||
)
|
||||
|
||||
def create_engine_config(
|
||||
self,
|
||||
usage_context: UsageContext | None = None,
|
||||
@@ -2263,19 +2284,7 @@ class EngineArgs:
|
||||
self.reasoning_parser_plugin
|
||||
)
|
||||
|
||||
observability_config = ObservabilityConfig(
|
||||
show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
|
||||
otlp_traces_endpoint=self.otlp_traces_endpoint,
|
||||
collect_detailed_traces=self.collect_detailed_traces,
|
||||
kv_cache_metrics=self.kv_cache_metrics,
|
||||
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
|
||||
cudagraph_metrics=self.cudagraph_metrics,
|
||||
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
|
||||
enable_mfu_metrics=self.enable_mfu_metrics,
|
||||
enable_mm_processor_stats=self.enable_mm_processor_stats,
|
||||
enable_logging_iteration_details=self.enable_logging_iteration_details,
|
||||
jit_monitor_verbose=self.jit_monitor_verbose,
|
||||
)
|
||||
observability_config = self.create_observability_config()
|
||||
|
||||
# Compilation config overrides
|
||||
compilation_config = copy.deepcopy(self.compilation_config)
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Monitor unexpected Triton kernel JIT compilation during inference.
|
||||
|
||||
After server warmup completes, any Triton JIT compilation or autotuning
|
||||
event indicates a cache miss or unexpected input shape that causes a
|
||||
latency spike. This module registers hooks in the Triton runtime to
|
||||
detect and log such events so they can be investigated.
|
||||
|
||||
Set ``--jit-monitor-verbose`` to log every Triton JIT compile with its
|
||||
dispatch key. This is intentionally opt-in because it can emit many logs and
|
||||
add overhead.
|
||||
|
||||
Currently monitors:
|
||||
- Triton ``@triton.autotune`` cache misses (via ``knobs.autotuning.print``)
|
||||
- Triton ``@triton.jit`` first-time compilations
|
||||
(via ``knobs.runtime.jit_post_compile_hook``)
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils.importing import HAS_TRITON
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_active: bool = False
|
||||
_verbose: bool = False
|
||||
|
||||
|
||||
def is_active() -> bool:
|
||||
"""Return whether the JIT compilation monitor is currently active."""
|
||||
return _active
|
||||
|
||||
|
||||
def activate(*, verbose: bool = False) -> None:
|
||||
"""Enable JIT compilation monitoring after warmup.
|
||||
|
||||
Call once per worker process at the end of
|
||||
:func:`compile_or_warm_up_model`. After activation every Triton
|
||||
kernel compilation or autotuning benchmark that happens during
|
||||
inference will be logged as a warning.
|
||||
|
||||
Safe to call multiple times — subsequent calls are no-ops.
|
||||
|
||||
If the user has explicitly set ``TRITON_PRINT_AUTOTUNING=0`` in
|
||||
their environment, autotuning printing is left disabled; the JIT
|
||||
compilation hook is still registered regardless.
|
||||
"""
|
||||
global _active, _verbose
|
||||
if _active:
|
||||
return
|
||||
_active = True
|
||||
_verbose = verbose
|
||||
|
||||
_setup_triton_autotuning_print()
|
||||
_setup_triton_jit_hook()
|
||||
|
||||
logger.info(
|
||||
"Kernel JIT monitor activated — Triton JIT compilations "
|
||||
"during inference will be logged as warnings."
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Triton autotuning print
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _setup_triton_autotuning_print() -> None:
|
||||
"""Enable ``TRITON_PRINT_AUTOTUNING`` unless the user opted out."""
|
||||
if not HAS_TRITON:
|
||||
return
|
||||
from triton import knobs # type: ignore[import-untyped]
|
||||
|
||||
user_val = os.environ.get("TRITON_PRINT_AUTOTUNING")
|
||||
if user_val == "0":
|
||||
logger.debug(
|
||||
"TRITON_PRINT_AUTOTUNING=0 set by user — "
|
||||
"autotuning messages will stay suppressed."
|
||||
)
|
||||
return
|
||||
|
||||
knobs.autotuning.print = True
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Triton JIT compilation hook
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _log_jit_compile(fn_name: str, kwargs) -> None:
|
||||
if _verbose:
|
||||
compile_info = kwargs.get("compile")
|
||||
if not isinstance(compile_info, dict):
|
||||
compile_info = {}
|
||||
logger.warning(
|
||||
"Triton %sJIT compilation during inference: %s (key=%s).",
|
||||
"autotune/warmup candidate " if kwargs.get("warmup") else "kernel ",
|
||||
fn_name,
|
||||
compile_info.get("key") or kwargs.get("key"),
|
||||
)
|
||||
return
|
||||
|
||||
logger.warning_once(
|
||||
"Triton kernel JIT compilation during inference: %s. "
|
||||
"This causes a latency spike; consider extending warmup "
|
||||
"to cover this shape/config.",
|
||||
fn_name,
|
||||
)
|
||||
|
||||
|
||||
def _setup_triton_jit_hook() -> None:
|
||||
"""Register a ``jit_post_compile_hook`` that warns on compilation."""
|
||||
if not HAS_TRITON:
|
||||
return
|
||||
from triton import knobs # type: ignore[import-untyped]
|
||||
|
||||
existing_hook = knobs.runtime.jit_post_compile_hook
|
||||
|
||||
def _on_jit_compile(**kwargs):
|
||||
# `jit_post_compile_hook` is Triton internal API and its
|
||||
# signature has changed across releases (kwargs added/renamed).
|
||||
# Accept **kwargs so an upstream change cannot crash this hook
|
||||
# with TypeError, and forward the full kwarg set to any
|
||||
# pre-existing hook unchanged.
|
||||
fn = kwargs.get("fn")
|
||||
fn_name = getattr(fn, "name", "<unknown>")
|
||||
_log_jit_compile(fn_name, kwargs)
|
||||
if existing_hook is not None:
|
||||
return existing_hook(**kwargs)
|
||||
return None
|
||||
|
||||
knobs.runtime.jit_post_compile_hook = _on_jit_compile
|
||||
@@ -0,0 +1,212 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Monitor unexpected kernel JIT compilation during inference.
|
||||
|
||||
After server warmup completes, any kernel JIT compilation or autotuning event
|
||||
indicates a cache miss or unexpected input shape that causes a latency spike.
|
||||
This module registers hooks in supported runtimes to detect such events so
|
||||
they can be investigated.
|
||||
|
||||
Set ``--jit-monitor-mode=error`` to fail fast on unexpected runtime
|
||||
compilation. Set ``--jit-monitor-verbose`` to log every JIT compile with
|
||||
additional runtime details. Verbose logging is intentionally opt-in because it
|
||||
can emit many logs and add overhead.
|
||||
|
||||
Currently monitors:
|
||||
- CuTeDSL cute.compile calls
|
||||
- Triton ``@triton.autotune`` cache misses (via ``knobs.autotuning.print``)
|
||||
- Triton ``@triton.jit`` first-time compilations
|
||||
(via ``knobs.runtime.jit_post_compile_hook``)
|
||||
"""
|
||||
|
||||
import functools
|
||||
import os
|
||||
from typing import Literal
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils.importing import HAS_TRITON
|
||||
|
||||
logger = init_logger(__name__)
|
||||
JitMonitorMode = Literal["warn", "error"]
|
||||
|
||||
_active: bool = False
|
||||
_mode: JitMonitorMode = "warn"
|
||||
_verbose: bool = False
|
||||
_cutedsl_hook_installed: bool = False
|
||||
|
||||
|
||||
def is_active() -> bool:
|
||||
"""Return whether the JIT compilation monitor is currently active."""
|
||||
return _active
|
||||
|
||||
|
||||
def activate(*, mode: JitMonitorMode = "warn", verbose: bool = False) -> None:
|
||||
"""Enable JIT compilation monitoring after warmup.
|
||||
|
||||
Call once per worker process at the end of
|
||||
:func:`compile_or_warm_up_model`. After activation every monitored kernel
|
||||
compilation or autotuning benchmark that happens during inference will be
|
||||
logged as a warning or raised as an error, depending on ``mode``.
|
||||
|
||||
Safe to call multiple times; subsequent calls are no-ops.
|
||||
|
||||
If the user has explicitly set ``TRITON_PRINT_AUTOTUNING=0`` in
|
||||
their environment, autotuning printing is left disabled; the JIT
|
||||
compilation hook is still registered regardless.
|
||||
"""
|
||||
global _active, _mode, _verbose
|
||||
if _active:
|
||||
return
|
||||
if mode not in ("warn", "error"):
|
||||
raise ValueError(f"Unsupported JIT monitor mode: {mode!r}")
|
||||
_active = True
|
||||
_mode = mode
|
||||
_verbose = verbose
|
||||
|
||||
_setup_triton_autotuning_print()
|
||||
_setup_triton_jit_hook()
|
||||
_setup_cutedsl_jit_hook()
|
||||
|
||||
logger.info(
|
||||
"Kernel JIT monitor activated; monitored JIT compilations during "
|
||||
"inference will use mode=%s.",
|
||||
mode,
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Triton autotuning print
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _setup_triton_autotuning_print() -> None:
|
||||
"""Enable ``TRITON_PRINT_AUTOTUNING`` unless the user opted out."""
|
||||
if not HAS_TRITON:
|
||||
return
|
||||
from triton import knobs # type: ignore[import-untyped]
|
||||
|
||||
user_val = os.environ.get("TRITON_PRINT_AUTOTUNING")
|
||||
if user_val == "0":
|
||||
logger.debug(
|
||||
"TRITON_PRINT_AUTOTUNING=0 set by user; "
|
||||
"autotuning messages will stay suppressed."
|
||||
)
|
||||
return
|
||||
|
||||
knobs.autotuning.print = True
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Triton JIT compilation hook
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _handle_jit_event(
|
||||
*,
|
||||
backend: str,
|
||||
event: str,
|
||||
fn_name: str,
|
||||
detail: str | None = None,
|
||||
) -> None:
|
||||
message = (
|
||||
"%s %s during inference: %s%s. "
|
||||
"This causes a latency spike; consider extending warmup "
|
||||
"to cover this shape/config."
|
||||
)
|
||||
detail_suffix = f" ({detail})" if detail else ""
|
||||
args = (backend, event, fn_name, detail_suffix)
|
||||
|
||||
if _mode == "error":
|
||||
raise RuntimeError(message % args)
|
||||
|
||||
if _verbose:
|
||||
logger.warning(message, *args)
|
||||
return
|
||||
|
||||
logger.warning_once(message, *args)
|
||||
|
||||
|
||||
def _log_triton_jit_compile(fn_name: str, kwargs) -> None:
|
||||
compile_info = kwargs.get("compile")
|
||||
if not isinstance(compile_info, dict):
|
||||
compile_info = {}
|
||||
key = compile_info.get("key") or kwargs.get("key")
|
||||
detail = f"key={key}" if _verbose and key is not None else None
|
||||
event = (
|
||||
"autotune/warmup candidate JIT compilation"
|
||||
if kwargs.get("warmup")
|
||||
else "kernel JIT compilation"
|
||||
)
|
||||
_handle_jit_event(
|
||||
backend="Triton",
|
||||
event=event,
|
||||
fn_name=fn_name,
|
||||
detail=detail,
|
||||
)
|
||||
|
||||
|
||||
def _setup_triton_jit_hook() -> None:
|
||||
"""Register a ``jit_post_compile_hook`` that warns on compilation."""
|
||||
if not HAS_TRITON:
|
||||
return
|
||||
from triton import knobs # type: ignore[import-untyped]
|
||||
|
||||
existing_hook = knobs.runtime.jit_post_compile_hook
|
||||
|
||||
def _on_jit_compile(**kwargs):
|
||||
# `jit_post_compile_hook` is Triton internal API and its
|
||||
# signature has changed across releases (kwargs added/renamed).
|
||||
# Accept **kwargs so an upstream change cannot crash this hook
|
||||
# with TypeError, and forward the full kwarg set to any
|
||||
# pre-existing hook unchanged.
|
||||
fn = kwargs.get("fn")
|
||||
fn_name = getattr(fn, "name", "<unknown>")
|
||||
_log_triton_jit_compile(fn_name, kwargs)
|
||||
if existing_hook is not None:
|
||||
return existing_hook(**kwargs)
|
||||
return None
|
||||
|
||||
knobs.runtime.jit_post_compile_hook = _on_jit_compile
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CuTeDSL JIT compilation hook
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _log_cutedsl_jit_compile(fn_name: str) -> None:
|
||||
_handle_jit_event(
|
||||
backend="CuTeDSL",
|
||||
event="JIT compilation",
|
||||
fn_name=fn_name,
|
||||
)
|
||||
|
||||
|
||||
def _setup_cutedsl_jit_hook() -> None:
|
||||
"""Wrap ``cutlass.cute.compile`` to warn on compilation."""
|
||||
global _cutedsl_hook_installed
|
||||
if _cutedsl_hook_installed:
|
||||
return
|
||||
|
||||
try:
|
||||
import cutlass.cute as cute
|
||||
except Exception:
|
||||
logger.debug("CuTeDSL is not available; skipping CuTeDSL JIT monitor.")
|
||||
return
|
||||
|
||||
original_compile = cute.compile
|
||||
|
||||
@functools.wraps(original_compile)
|
||||
def _compile_with_monitor(*args, **kwargs):
|
||||
kernel = args[0] if args else kwargs.get("function")
|
||||
kernel_name = getattr(kernel, "__name__", None)
|
||||
if kernel_name is None:
|
||||
kernel_name = (
|
||||
kernel.__class__.__name__ if kernel is not None else "<unknown>"
|
||||
)
|
||||
_log_cutedsl_jit_compile(kernel_name)
|
||||
return original_compile(*args, **kwargs)
|
||||
|
||||
cute.compile = _compile_with_monitor
|
||||
_cutedsl_hook_installed = True
|
||||
@@ -761,12 +761,11 @@ class Worker(WorkerBase):
|
||||
|
||||
# All warmup is done — start monitoring for unexpected JIT
|
||||
# compilations that would cause latency spikes during inference.
|
||||
from vllm.triton_utils.jit_monitor import (
|
||||
activate as activate_triton_jit_monitor,
|
||||
)
|
||||
from vllm.utils.jit_monitor import activate as activate_jit_monitor
|
||||
|
||||
activate_triton_jit_monitor(
|
||||
verbose=self.observability_config.jit_monitor_verbose
|
||||
activate_jit_monitor(
|
||||
mode=self.observability_config.jit_monitor_mode,
|
||||
verbose=self.observability_config.jit_monitor_verbose,
|
||||
)
|
||||
|
||||
# Freeze the worker heap so the GC won't scan static objects
|
||||
|
||||
Reference in New Issue
Block a user