diff --git a/docs/design/debug_vllm_compile.md b/docs/design/debug_vllm_compile.md index 7edda6fa64..af9b3f530d 100644 --- a/docs/design/debug_vllm_compile.md +++ b/docs/design/debug_vllm_compile.md @@ -251,6 +251,18 @@ that may call 1+ triton kernels. On rare (but unfortunate) occasions, it may produce an incorrect triton kernel. This may manifest as silent incorrectness, CUDA illegal memory accesses, or loud errors. +### Deterministic compilation + +When vLLM selects combo-kernel defaults, it disables combo-kernel benchmarking +if Inductor's `deterministic` setting is enabled. A `deterministic` value in +`inductor_compile_config` takes precedence over the global Inductor setting. +On torch 2.13, `TORCHINDUCTOR_DETERMINISTIC=1` alone is reset by the first +compiled frame ([pytorch#198563](https://github.com/pytorch/pytorch/issues/198563)), +so set `deterministic` in `inductor_compile_config` instead. +Explicit `combo_kernels` or `benchmark_combo_kernel` settings are preserved. +This avoids conflicting compilation defaults; it does not guarantee reproducible +model outputs. + ### Inductor runtime assertions By default (on torch < 2.12), vLLM disables Inductor's runtime assertions diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index 24cb816b5d..5a1ba13a57 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -1366,3 +1366,70 @@ def test_inductor_asserts_user_override(monkeypatch): assert config.inductor_compile_config.get("size_asserts") is True if not _is_torch_equal_or_newer(torch.__version__, "2.12.0.dev"): assert config.inductor_compile_config.get("alignment_asserts") is False + + +@pytest.mark.parametrize("deterministic", [False, True]) +@pytest.mark.parametrize("override", [None, False, True]) +def test_combo_kernel_benchmarking_respects_deterministic(deterministic, override): + from torch._inductor import config as inductor_config + + overrides = {} if override is None else {"deterministic": override} + with ( + inductor_config.patch(deterministic=deterministic), + patch("vllm.config.compilation.current_platform.is_cpu", return_value=False), + ): + config = CompilationConfig(inductor_compile_config=overrides) + + assert config.inductor_compile_config["combo_kernels"] is True + effective_deterministic = deterministic if override is None else override + assert config.inductor_compile_config["benchmark_combo_kernel"] is ( + not effective_deterministic + ) + + +@pytest.mark.parametrize("deterministic", [False, True]) +@pytest.mark.parametrize( + "overrides", + [ + {"combo_kernels": False}, + {"combo_kernels": True}, + {"benchmark_combo_kernel": False}, + {"benchmark_combo_kernel": True}, + ], +) +def test_combo_kernel_explicit_settings_preserved(deterministic, overrides): + from torch._inductor import config as inductor_config + + with ( + inductor_config.patch(deterministic=deterministic), + patch("vllm.config.compilation.current_platform.is_cpu", return_value=False), + ): + config = CompilationConfig(inductor_compile_config=overrides.copy()) + + for key in ("combo_kernels", "benchmark_combo_kernel"): + assert config.inductor_compile_config.get(key) == overrides.get(key) + + +@pytest.mark.parametrize("is_cpu,torch_version", [(True, "2.13.0"), (False, "2.8.0")]) +def test_combo_kernel_defaults_require_supported_platform_and_torch( + is_cpu, torch_version +): + with ( + patch("vllm.config.compilation.current_platform.is_cpu", return_value=is_cpu), + patch("torch.__version__", torch_version), + ): + config = CompilationConfig() + + assert "combo_kernels" not in config.inductor_compile_config + assert "benchmark_combo_kernel" not in config.inductor_compile_config + + +def test_combo_kernel_defaults_without_inductor_deterministic_setting(): + with ( + patch("torch._inductor.config", SimpleNamespace()), + patch("vllm.config.compilation.current_platform.is_cpu", return_value=False), + ): + config = CompilationConfig() + + assert config.inductor_compile_config["combo_kernels"] is True + assert config.inductor_compile_config["benchmark_combo_kernel"] is True diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index afe576daa9..cbd132ab5b 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -1002,10 +1002,16 @@ class CompilationConfig: # (fixme @boyuan) combo kernel does not support cpu yet. and not current_platform.is_cpu() ): + from torch._inductor import config as inductor_config + # use horizontal fusion, which is useful for fusing qk-norm and # qk-rope when query and key have different shapes. self.inductor_compile_config["combo_kernels"] = True - self.inductor_compile_config["benchmark_combo_kernel"] = True + + deterministic = self.inductor_compile_config.get( + "deterministic", getattr(inductor_config, "deterministic", False) + ) + self.inductor_compile_config["benchmark_combo_kernel"] = not deterministic if self.use_inductor_graph_partition and not is_torch_equal_or_newer( "2.9.0.dev"