[CI][Bugfix] Make test_gpt2_cache_hit observable across V1 EngineCore (#42037)
Signed-off-by: haosdent <haosdent@gmail.com>
This commit is contained in:
@@ -1,9 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import functools
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import multiprocessing
|
|
||||||
import os
|
import os
|
||||||
import pickle
|
import pickle
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -15,7 +13,6 @@ import pytest
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
import vllm.model_executor.layers.activation
|
|
||||||
from vllm.compilation.backends import VllmBackend
|
from vllm.compilation.backends import VllmBackend
|
||||||
from vllm.compilation.caching import (
|
from vllm.compilation.caching import (
|
||||||
StandaloneCompiledArtifacts,
|
StandaloneCompiledArtifacts,
|
||||||
@@ -476,64 +473,57 @@ def test_standalone_compile_correctness():
|
|||||||
@create_new_process_for_each_test("spawn")
|
@create_new_process_for_each_test("spawn")
|
||||||
def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch):
|
def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch):
|
||||||
"""
|
"""
|
||||||
Test that compiling gpt2 twice results in a cache hit and
|
Test that compiling gpt2 twice results in a cache hit.
|
||||||
capture torch dynamic symbol creations to ensure make_symbol
|
|
||||||
not called on cache hit.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import torch.fx.experimental.symbolic_shapes as symbolic_shapes_module
|
Counter values are read from the EngineCore subprocess via
|
||||||
from torch.utils._sympy.symbol import make_symbol
|
``LLM.collective_rpc`` so the test works under default V1
|
||||||
|
multiprocessing (no shared memory between test and engine).
|
||||||
|
"""
|
||||||
|
|
||||||
from vllm import LLM
|
from vllm import LLM
|
||||||
|
|
||||||
create_symbol_counter = multiprocessing.Value("i", 0)
|
def _snap(self):
|
||||||
original_make_symbol = make_symbol
|
from vllm.compilation.counter import compilation_counter
|
||||||
|
|
||||||
@functools.wraps(original_make_symbol)
|
return (
|
||||||
def counting_make_symbol(prefix, idx, **kwargs):
|
compilation_counter.num_aot_compiles,
|
||||||
with create_symbol_counter.get_lock():
|
compilation_counter.num_aot_artifacts_saved,
|
||||||
create_symbol_counter.value += 1
|
compilation_counter.num_aot_artifacts_loaded,
|
||||||
return original_make_symbol(prefix, idx, **kwargs)
|
)
|
||||||
|
|
||||||
symbolic_shapes_module.make_symbol = counting_make_symbol
|
# collective_rpc(callable) requires pickle-based serialization.
|
||||||
try:
|
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||||
with monkeypatch.context() as m, tempfile.TemporaryDirectory() as tmpdirname:
|
|
||||||
m.setenv("VLLM_CACHE_ROOT", tmpdirname)
|
|
||||||
m.setenv("VLLM_USE_AOT_COMPILE", "1")
|
|
||||||
# First compilation - initialize model and generate
|
|
||||||
llm_model = LLM(
|
|
||||||
model="gpt2",
|
|
||||||
compilation_config=CompilationConfig(
|
|
||||||
mode=CompilationMode.VLLM_COMPILE,
|
|
||||||
),
|
|
||||||
max_model_len=256,
|
|
||||||
)
|
|
||||||
|
|
||||||
llm_model.generate("Hello, my name is")
|
with monkeypatch.context() as m, tempfile.TemporaryDirectory() as tmpdirname:
|
||||||
assert create_symbol_counter.value == 2
|
m.setenv("VLLM_CACHE_ROOT", tmpdirname)
|
||||||
create_symbol_counter.value = 0
|
m.setenv("VLLM_USE_AOT_COMPILE", "1")
|
||||||
|
# First compilation - initialize model and generate
|
||||||
|
llm_model = LLM(
|
||||||
|
model="gpt2",
|
||||||
|
compilation_config=CompilationConfig(
|
||||||
|
mode=CompilationMode.VLLM_COMPILE,
|
||||||
|
),
|
||||||
|
max_model_len=256,
|
||||||
|
)
|
||||||
|
|
||||||
# Clean up first model
|
llm_model.generate("Hello, my name is")
|
||||||
del llm_model
|
assert llm_model.collective_rpc(_snap)[0] == (1, 1, 0)
|
||||||
disable_envs_cache()
|
|
||||||
vllm.model_executor.layers.activation._ACTIVATION_REGISTRY._dict.clear()
|
|
||||||
|
|
||||||
# Second compilation - should hit cache
|
# Clean up first model
|
||||||
m.setenv("VLLM_FORCE_AOT_LOAD", "1")
|
del llm_model
|
||||||
llm_model = LLM(
|
disable_envs_cache()
|
||||||
model="gpt2",
|
|
||||||
compilation_config=CompilationConfig(
|
|
||||||
mode=CompilationMode.VLLM_COMPILE,
|
|
||||||
),
|
|
||||||
max_model_len=256,
|
|
||||||
)
|
|
||||||
llm_model.generate("Hello, my name is")
|
|
||||||
|
|
||||||
assert create_symbol_counter.value == 0
|
# Second compilation - should hit cache
|
||||||
|
m.setenv("VLLM_FORCE_AOT_LOAD", "1")
|
||||||
finally:
|
llm_model = LLM(
|
||||||
# Restore original method
|
model="gpt2",
|
||||||
symbolic_shapes_module.make_symbol = original_make_symbol
|
compilation_config=CompilationConfig(
|
||||||
|
mode=CompilationMode.VLLM_COMPILE,
|
||||||
|
),
|
||||||
|
max_model_len=256,
|
||||||
|
)
|
||||||
|
llm_model.generate("Hello, my name is")
|
||||||
|
assert llm_model.collective_rpc(_snap)[0] == (0, 0, 1)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||||
|
|||||||
Reference in New Issue
Block a user