[Bugfix] Respect Inductor deterministic mode in combo-kernel defaults (#59074)
Signed-off-by: cheesecake <farzanaman99@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5.5
parent
32fbfa15e8
commit
b9767299db
@@ -251,6 +251,18 @@ that may call 1+ triton kernels. On rare (but unfortunate) occasions, it may
|
||||
produce an incorrect triton kernel. This may manifest as silent incorrectness,
|
||||
CUDA illegal memory accesses, or loud errors.
|
||||
|
||||
### Deterministic compilation
|
||||
|
||||
When vLLM selects combo-kernel defaults, it disables combo-kernel benchmarking
|
||||
if Inductor's `deterministic` setting is enabled. A `deterministic` value in
|
||||
`inductor_compile_config` takes precedence over the global Inductor setting.
|
||||
On torch 2.13, `TORCHINDUCTOR_DETERMINISTIC=1` alone is reset by the first
|
||||
compiled frame ([pytorch#198563](https://github.com/pytorch/pytorch/issues/198563)),
|
||||
so set `deterministic` in `inductor_compile_config` instead.
|
||||
Explicit `combo_kernels` or `benchmark_combo_kernel` settings are preserved.
|
||||
This avoids conflicting compilation defaults; it does not guarantee reproducible
|
||||
model outputs.
|
||||
|
||||
### Inductor runtime assertions
|
||||
|
||||
By default (on torch < 2.12), vLLM disables Inductor's runtime assertions
|
||||
|
||||
@@ -1366,3 +1366,70 @@ def test_inductor_asserts_user_override(monkeypatch):
|
||||
assert config.inductor_compile_config.get("size_asserts") is True
|
||||
if not _is_torch_equal_or_newer(torch.__version__, "2.12.0.dev"):
|
||||
assert config.inductor_compile_config.get("alignment_asserts") is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deterministic", [False, True])
|
||||
@pytest.mark.parametrize("override", [None, False, True])
|
||||
def test_combo_kernel_benchmarking_respects_deterministic(deterministic, override):
|
||||
from torch._inductor import config as inductor_config
|
||||
|
||||
overrides = {} if override is None else {"deterministic": override}
|
||||
with (
|
||||
inductor_config.patch(deterministic=deterministic),
|
||||
patch("vllm.config.compilation.current_platform.is_cpu", return_value=False),
|
||||
):
|
||||
config = CompilationConfig(inductor_compile_config=overrides)
|
||||
|
||||
assert config.inductor_compile_config["combo_kernels"] is True
|
||||
effective_deterministic = deterministic if override is None else override
|
||||
assert config.inductor_compile_config["benchmark_combo_kernel"] is (
|
||||
not effective_deterministic
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deterministic", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[
|
||||
{"combo_kernels": False},
|
||||
{"combo_kernels": True},
|
||||
{"benchmark_combo_kernel": False},
|
||||
{"benchmark_combo_kernel": True},
|
||||
],
|
||||
)
|
||||
def test_combo_kernel_explicit_settings_preserved(deterministic, overrides):
|
||||
from torch._inductor import config as inductor_config
|
||||
|
||||
with (
|
||||
inductor_config.patch(deterministic=deterministic),
|
||||
patch("vllm.config.compilation.current_platform.is_cpu", return_value=False),
|
||||
):
|
||||
config = CompilationConfig(inductor_compile_config=overrides.copy())
|
||||
|
||||
for key in ("combo_kernels", "benchmark_combo_kernel"):
|
||||
assert config.inductor_compile_config.get(key) == overrides.get(key)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_cpu,torch_version", [(True, "2.13.0"), (False, "2.8.0")])
|
||||
def test_combo_kernel_defaults_require_supported_platform_and_torch(
|
||||
is_cpu, torch_version
|
||||
):
|
||||
with (
|
||||
patch("vllm.config.compilation.current_platform.is_cpu", return_value=is_cpu),
|
||||
patch("torch.__version__", torch_version),
|
||||
):
|
||||
config = CompilationConfig()
|
||||
|
||||
assert "combo_kernels" not in config.inductor_compile_config
|
||||
assert "benchmark_combo_kernel" not in config.inductor_compile_config
|
||||
|
||||
|
||||
def test_combo_kernel_defaults_without_inductor_deterministic_setting():
|
||||
with (
|
||||
patch("torch._inductor.config", SimpleNamespace()),
|
||||
patch("vllm.config.compilation.current_platform.is_cpu", return_value=False),
|
||||
):
|
||||
config = CompilationConfig()
|
||||
|
||||
assert config.inductor_compile_config["combo_kernels"] is True
|
||||
assert config.inductor_compile_config["benchmark_combo_kernel"] is True
|
||||
|
||||
@@ -1002,10 +1002,16 @@ class CompilationConfig:
|
||||
# (fixme @boyuan) combo kernel does not support cpu yet.
|
||||
and not current_platform.is_cpu()
|
||||
):
|
||||
from torch._inductor import config as inductor_config
|
||||
|
||||
# use horizontal fusion, which is useful for fusing qk-norm and
|
||||
# qk-rope when query and key have different shapes.
|
||||
self.inductor_compile_config["combo_kernels"] = True
|
||||
self.inductor_compile_config["benchmark_combo_kernel"] = True
|
||||
|
||||
deterministic = self.inductor_compile_config.get(
|
||||
"deterministic", getattr(inductor_config, "deterministic", False)
|
||||
)
|
||||
self.inductor_compile_config["benchmark_combo_kernel"] = not deterministic
|
||||
|
||||
if self.use_inductor_graph_partition and not is_torch_equal_or_newer(
|
||||
"2.9.0.dev"
|
||||
|
||||
Reference in New Issue
Block a user