[AOT compilation] support torch.compile inductor artifacts in VllmCompiledFunction (#25205)

Signed-off-by: dolpm <34420038+dolpm@users.noreply.github.com>
This commit is contained in:
dolpm
2026-01-20 19:45:59 +00:00
committed by GitHub
parent 193069d129
commit 7c5dedc247
8 changed files with 1169 additions and 113 deletions
+415 -2
View File
@@ -2,15 +2,21 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import hashlib
import multiprocessing
import pickle
import tempfile
from contextlib import contextmanager
from pathlib import Path
from unittest.mock import Mock, patch
import pytest
import torch
import vllm.model_executor.layers.activation
from vllm.compilation.caching import (
StandaloneCompiledArtifacts,
)
from vllm.compilation.decorators import support_torch_compile
from vllm.config import (
CompilationConfig,
@@ -40,6 +46,15 @@ def reference_fn(x: torch.Tensor):
return x
def reference_fn_tuple(x: torch.Tensor):
"""Reference function that returns a tuple of tensors."""
assert x.shape[0] <= 42
assert x.shape[0] % 2 == 0
for _ in range(3000):
x = x + x.shape[0]
return x, x * 2
@support_torch_compile
class CompiledMod(torch.nn.Module):
def __init__(self, **kwargs):
@@ -49,10 +64,22 @@ class CompiledMod(torch.nn.Module):
return reference_fn(x)
@support_torch_compile
class CompiledModTuple(torch.nn.Module):
"""A compiled module that returns a tuple of tensors."""
def __init__(self, **kwargs):
super().__init__()
def forward(self, x: torch.Tensor):
return reference_fn_tuple(x)
def make_vllm_config() -> VllmConfig:
return VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
backend="inductor",
)
)
@@ -73,6 +100,8 @@ def test_no_dynamo_cache_entry(monkeypatch: pytest.MonkeyPatch):
expected = reference_fn(*args)
with use_vllm_config(vllm_config):
m.setenv("VLLM_USE_AOT_COMPILE", "0")
m.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
m.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
with (
pytest.raises(RuntimeError, match="Detected recompile"),
torch.compiler.set_stance("fail_on_recompile"),
@@ -94,6 +123,8 @@ def test_force_aot_load(monkeypatch: pytest.MonkeyPatch):
with tempfile.TemporaryDirectory() as tmpdirname, monkeypatch.context() as m:
args = (torch.randn(10, 10),)
m.setenv("VLLM_USE_AOT_COMPILE", "1")
m.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
m.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
m.setenv("VLLM_FORCE_AOT_LOAD", "1")
m.setenv("VLLM_CACHE_ROOT", tmpdirname)
vllm_config = make_vllm_config()
@@ -111,18 +142,158 @@ def test_save_and_load(monkeypatch: pytest.MonkeyPatch):
with tempfile.TemporaryDirectory() as tmpdirname:
m.setenv("VLLM_CACHE_ROOT", tmpdirname)
m.setenv("VLLM_USE_AOT_COMPILE", "1")
m.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
m.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
vllm_config = make_vllm_config()
with use_vllm_config(vllm_config):
expected = CompiledMod(vllm_config=vllm_config)(*args)
compiled_mod = CompiledMod(vllm_config=vllm_config)
expected = compiled_mod(*args)
disable_envs_cache()
m.setenv("VLLM_FORCE_AOT_LOAD", "1")
vllm_config = make_vllm_config()
with use_vllm_config(vllm_config):
ret = CompiledMod(vllm_config=vllm_config)(*args)
cached_mod = CompiledMod(vllm_config=vllm_config)
ret = cached_mod(*args)
assert cached_mod.was_aot_compile_fn_loaded_from_disk, (
"Expected was_aot_compile_fn_loaded_from_disk to be True"
)
assert torch.allclose(ret, expected)
@pytest.mark.skipif(
not is_torch_equal_or_newer("2.10.0.dev"), reason="requires torch 2.10"
)
def test_cache_load_returns_tuple_consistency(monkeypatch: pytest.MonkeyPatch):
"""
Test that cache loading correctly handles the returns_tuple logic.
This verifies that when a model returns a single tensor (not a tuple),
the output type is consistent between fresh compilation and cache load.
Without the fix, cached artifacts would return [tensor] instead of tensor.
"""
with monkeypatch.context() as m:
args = (torch.randn(10, 10),)
with tempfile.TemporaryDirectory() as tmpdirname:
m.setenv("VLLM_CACHE_ROOT", tmpdirname)
m.setenv("VLLM_USE_AOT_COMPILE", "1")
m.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
m.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
vllm_config = make_vllm_config()
# Fresh compilation
with use_vllm_config(vllm_config):
compiled_mod = CompiledMod(vllm_config=vllm_config)
fresh_result = compiled_mod(*args)
fresh_result_type = type(fresh_result)
# Verify fresh result is a tensor, not a tuple/list
assert isinstance(fresh_result, torch.Tensor), (
f"Fresh compile should return tensor, got {fresh_result_type}"
)
disable_envs_cache()
# Load from cache
m.setenv("VLLM_FORCE_AOT_LOAD", "1")
vllm_config = make_vllm_config()
with use_vllm_config(vllm_config):
cached_mod = CompiledMod(vllm_config=vllm_config)
cached_result = cached_mod(*args)
cached_result_type = type(cached_result)
# Verify cache was actually loaded
assert cached_mod.was_aot_compile_fn_loaded_from_disk, (
"Expected was_aot_compile_fn_loaded_from_disk to be True after "
"loading from cache"
)
# Verify cached result has same type as fresh result
assert isinstance(cached_result, torch.Tensor), (
f"Cache load should return tensor, got {cached_result_type}. "
"This indicates the returns_tuple logic is not being applied "
"correctly when loading from cache."
)
# Verify values match
assert torch.allclose(cached_result, fresh_result), (
"Cached result values should match fresh compilation"
)
@pytest.mark.skipif(
not is_torch_equal_or_newer("2.10.0.dev"), reason="requires torch 2.10"
)
def test_cache_load_returns_tuple_consistency_tuple_output(
monkeypatch: pytest.MonkeyPatch,
):
"""
Test that cache loading correctly handles models that return tuples.
This verifies that when a model returns a tuple of tensors, the output
type is preserved as a tuple between fresh compilation and cache load.
"""
with monkeypatch.context() as m:
args = (torch.randn(10, 10),)
with tempfile.TemporaryDirectory() as tmpdirname:
m.setenv("VLLM_CACHE_ROOT", tmpdirname)
m.setenv("VLLM_USE_AOT_COMPILE", "1")
m.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
m.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
vllm_config = make_vllm_config()
# Fresh compilation with tuple-returning model
with use_vllm_config(vllm_config):
compiled_mod = CompiledModTuple(vllm_config=vllm_config)
fresh_result = compiled_mod(*args)
fresh_result_type = type(fresh_result)
# Verify fresh result is a tuple
assert isinstance(fresh_result, tuple), (
f"Fresh compile should return tuple, got {fresh_result_type}"
)
assert len(fresh_result) == 2, (
f"Fresh compile should return 2-tuple, got {len(fresh_result)}"
)
disable_envs_cache()
# Load from cache
m.setenv("VLLM_FORCE_AOT_LOAD", "1")
vllm_config = make_vllm_config()
with use_vllm_config(vllm_config):
cached_mod = CompiledModTuple(vllm_config=vllm_config)
cached_result = cached_mod(*args)
cached_result_type = type(cached_result)
# Verify cache was actually loaded
assert cached_mod.was_aot_compile_fn_loaded_from_disk, (
"Expected was_aot_compile_fn_loaded_from_disk to be True after "
"loading from cache"
)
# Verify cached result is also a tuple
assert isinstance(cached_result, tuple), (
f"Cache load should return tuple, got {cached_result_type}. "
"This indicates the returns_tuple logic is not preserving "
"tuple outputs when loading from cache."
)
assert len(cached_result) == 2, (
f"Cache load should return 2-tuple, got {len(cached_result)}"
)
# Verify values match
assert torch.allclose(cached_result[0], fresh_result[0]), (
"Cached result[0] values should match fresh compilation"
)
assert torch.allclose(cached_result[1], fresh_result[1]), (
"Cached result[1] values should match fresh compilation"
)
@pytest.mark.skipif(
not is_torch_equal_or_newer("2.10.0.dev"), reason="requires torch 2.10"
)
@@ -137,6 +308,8 @@ def test_shape_env(monkeypatch: pytest.MonkeyPatch):
with tempfile.TemporaryDirectory() as tmpdirname:
m.setenv("VLLM_CACHE_ROOT", tmpdirname)
m.setenv("VLLM_USE_AOT_COMPILE", "1")
m.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", "1")
m.setenv("VLLM_USE_STANDALONE_COMPILE", "1")
vllm_config = make_vllm_config()
with use_vllm_config(vllm_config):
compiled_mod = CompiledMod(vllm_config=vllm_config)
@@ -144,6 +317,7 @@ def test_shape_env(monkeypatch: pytest.MonkeyPatch):
artifacts = compiled_mod.aot_compiled_fn._artifacts
guards_string = artifacts.compiled_fn.shape_env.format_guards()
assert guards_string == " - s77 <= 42\n - Eq(Mod(s77, 2), 0)"
disable_envs_cache()
m.setenv("VLLM_FORCE_AOT_LOAD", "1")
@@ -151,6 +325,9 @@ def test_shape_env(monkeypatch: pytest.MonkeyPatch):
with use_vllm_config(vllm_config):
compiled_mod = CompiledMod(vllm_config=vllm_config)
compiled_mod(*args)
assert compiled_mod.was_aot_compile_fn_loaded_from_disk, (
"Expected was_aot_compile_fn_loaded_from_disk to be True"
)
artifacts = compiled_mod.aot_compiled_fn._artifacts
guards_string = artifacts.compiled_fn.shape_env.format_guards()
assert guards_string == " - s77 <= 42\n - Eq(Mod(s77, 2), 0)"
@@ -188,6 +365,7 @@ def test_partition_wrapper_applied_on_aot_load(
with use_vllm_config(vllm_config):
compiled_mod = CompiledMod(vllm_config=vllm_config)
compiled_mod(*args)
disable_envs_cache()
# Second run - load from cache, verify partition wrapper applied
@@ -210,6 +388,11 @@ def test_partition_wrapper_applied_on_aot_load(
# This tests the fix for the first call after a restart.
compiled_mod(*args)
# Verify cache was loaded
assert compiled_mod.was_aot_compile_fn_loaded_from_disk, (
"Expected was_aot_compile_fn_loaded_from_disk to be True"
)
# Verify partition wrapper was called on AOT load.
assert spy.call_count >= 2, (
"Expected partition wrapper to be set and cleared on AOT load, "
@@ -307,3 +490,233 @@ def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch):
finally:
# Restore original method
symbolic_shapes_module.make_symbol = original_make_symbol
@pytest.mark.skipif(
not is_torch_equal_or_newer("2.10.0.dev"), reason="requires torch 2.10"
)
class TestStandaloneCompiledArtifacts:
def test_init(self):
cache = StandaloneCompiledArtifacts()
assert cache.submodule_bytes == {}
assert cache.submodule_bytes_store == {}
assert cache.loaded_submodule_store == {}
def test_insert_new_artifact(self):
cache = StandaloneCompiledArtifacts()
test_data = b"test_artifact_data"
submod_name = "test_submod"
shape = "s1"
hasher = hashlib.sha256()
hasher.update(test_data)
expected_hash = hasher.hexdigest()
cache.insert(submod_name, shape, test_data)
assert f"{submod_name}_{shape}" in cache.submodule_bytes
assert cache.submodule_bytes[f"{submod_name}_{shape}"] == expected_hash
assert expected_hash in cache.submodule_bytes_store
assert cache.submodule_bytes_store[expected_hash] == test_data
def test_insert_duplicate_artifact(self):
cache = StandaloneCompiledArtifacts()
test_data = b"duplicate_test_data"
submod_name1 = "submod1"
submod_name2 = "submod2"
shape = "s2"
cache.insert(submod_name1, shape, test_data)
cache.insert(submod_name2, shape, test_data)
hash1 = cache.submodule_bytes[f"{submod_name1}_{shape}"]
hash2 = cache.submodule_bytes[f"{submod_name2}_{shape}"]
assert hash1 == hash2
assert len(cache.submodule_bytes_store) == 1
assert len(cache.submodule_bytes) == 2
def test_get_artifact(self):
cache = StandaloneCompiledArtifacts()
test_data = b"retrievable_data"
submod_name = "mod1"
shape = "shape16"
cache.insert(submod_name, shape, test_data)
retrieved_data = cache.get(submod_name, shape)
assert retrieved_data == test_data
def test_get_nonexistent_artifact(self):
cache = StandaloneCompiledArtifacts()
with pytest.raises(KeyError):
cache.get("nonexistent", "shape")
def test_size_bytes(self):
cache = StandaloneCompiledArtifacts()
assert cache.size_bytes() == 0
data1 = b"x" * 100
data2 = b"y" * 200
cache.insert("mod1", "shape1", data1)
cache.insert("mod2", "shape2", data2)
assert cache.size_bytes() == 300
def test_num_artifacts_and_entries(self):
cache = StandaloneCompiledArtifacts()
assert cache.num_artifacts() == 0
assert cache.num_entries() == 0
cache.insert("mod1", "shape1", b"data1")
cache.insert("mod2", "shape2", b"data2")
assert cache.num_artifacts() == 2
assert cache.num_entries() == 2
cache.insert("mod3", "shape3", b"data1")
assert cache.num_artifacts() == 2
assert cache.num_entries() == 3
@patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize")
def test_load_all_success(self, mock_deserialize):
"""Test successful loading of all artifacts"""
cache = StandaloneCompiledArtifacts()
mock_artifact1 = Mock()
mock_artifact2 = Mock()
mock_deserialize.side_effect = [mock_artifact1, mock_artifact2]
cache.insert("mod1", "shape1", pickle.dumps(b"data1"))
cache.insert("mod2", "shape2", pickle.dumps(b"data2"))
cache.load_all()
assert len(cache.loaded_submodule_store) == 2
assert mock_deserialize.call_count == 2
@patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize")
def test_load_all_already_loaded(self, mock_deserialize):
"""Test that load_all skips if already loaded"""
cache = StandaloneCompiledArtifacts()
mock_artifact = Mock()
cache.submodule_bytes_store["hash1"] = pickle.dumps(b"data1")
cache.loaded_submodule_store["hash1"] = mock_artifact
cache.load_all()
mock_deserialize.assert_not_called()
@patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize")
def test_get_loaded_artifact(self, mock_deserialize):
"""Test retrieving loaded artifacts"""
cache = StandaloneCompiledArtifacts()
mock_artifact = Mock()
mock_deserialize.return_value = mock_artifact
submod_name = "test_mod"
shape = "test_shape"
cache.insert(submod_name, shape, pickle.dumps(b"test_data"))
cache.load_all()
retrieved_artifact = cache.get_loaded(submod_name, shape)
assert retrieved_artifact == mock_artifact
def test_getstate_setstate(self):
cache = StandaloneCompiledArtifacts()
cache.insert("mod1", "shape1", b"data1")
cache.insert("mod2", "shape2", b"data2")
cache.loaded_submodule_store["hash1"] = Mock()
state = cache.__getstate__()
assert "submodule_bytes" in state
assert "submodule_bytes_store" in state
assert "loaded_submodule_store" not in state
new_cache = StandaloneCompiledArtifacts()
new_cache.__setstate__(state)
assert new_cache.submodule_bytes == cache.submodule_bytes
assert new_cache.submodule_bytes_store == cache.submodule_bytes_store
assert new_cache.loaded_submodule_store == {}
def test_pickle_roundtrip(self):
cache = StandaloneCompiledArtifacts()
test_data1 = b"pickle_test_data_1"
test_data2 = b"pickle_test_data_2"
cache.insert("mod1", "shape1", test_data1)
cache.insert("mod2", "shape2", test_data2)
pickled_data = pickle.dumps(cache)
restored_cache = pickle.loads(pickled_data)
assert restored_cache.get("mod1", "shape1") == test_data1
assert restored_cache.get("mod2", "shape2") == test_data2
assert restored_cache.num_artifacts() == cache.num_artifacts()
assert restored_cache.num_entries() == cache.num_entries()
assert restored_cache.size_bytes() == cache.size_bytes()
assert len(restored_cache.loaded_submodule_store) == 0
@pytest.mark.skipif(
not is_torch_equal_or_newer("2.10.0.dev"), reason="requires torch 2.10"
)
class TestStandaloneCompiledArtifactsIntegration:
def test_add_pickle_unpickle(self):
cache = StandaloneCompiledArtifacts()
artifacts = {
("mod1", "shape1"): b"m1s1_artifact",
("mod1", "shape2"): b"m1s2_artifact",
("mod2", "shape1"): b"m2s1_artifact",
("mod2", "shape2"): b"m2s2_artifact",
}
for (submod, shape), data in artifacts.items():
cache.insert(submod, shape, data)
assert cache.num_entries() == 4
assert cache.num_artifacts() == 4
for (submod, shape), expected_data in artifacts.items():
retrieved_data = cache.get(submod, shape)
assert retrieved_data == expected_data
pickled = pickle.dumps(cache)
restored_cache = pickle.loads(pickled)
for (submod, shape), expected_data in artifacts.items():
retrieved_data = restored_cache.get(submod, shape)
assert retrieved_data == expected_data
def test_deduplication(self):
cache = StandaloneCompiledArtifacts()
shared_data = b"shared_artifact_data" * 1000
cache.insert("mod1", "shape1", shared_data)
cache.insert("mod2", "shape1", shared_data)
cache.insert("mod1", "shape2", shared_data)
cache.insert("mod3", "shape3", shared_data)
assert cache.num_entries() == 4
assert cache.num_artifacts() == 1
assert cache.size_bytes() == len(shared_data)
for submod, shape in [
("mod1", "shape1"),
("mod2", "shape1"),
("mod1", "shape2"),
("mod3", "shape3"),
]:
assert cache.get(submod, shape) == shared_data