[CI][Bugfix] Fix failure CI step "PyTorch Fullgraph Smoke Test" (#41953)
Signed-off-by: haosdent <haosdent@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
@@ -161,7 +161,14 @@ def _run_simple_model(
|
||||
@pytest.mark.parametrize("intermediate_unbacked", [True, False])
|
||||
@torch.inference_mode()
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_simple_piecewise_compile(backend, intermediate_unbacked):
|
||||
def test_simple_piecewise_compile(backend, intermediate_unbacked, monkeypatch):
|
||||
# `intermediate_unbacked` flips a control-flow branch inside
|
||||
# `SillyModel.forward`, but the AOT-compile cache key only hashes the
|
||||
# forward function's qualname + line number, so both parametrize variants
|
||||
# share the same cache slot. Disabling the cache forces each variant to
|
||||
# compile fresh; otherwise the second-running variant loads the first's
|
||||
# artifact and segfaults with an illegal memory access.
|
||||
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
|
||||
_run_simple_model(
|
||||
splitting_ops=["silly::attention"],
|
||||
use_inductor_graph_partition=False,
|
||||
|
||||
+45
-14
@@ -1511,43 +1511,72 @@ def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, Non
|
||||
return wrapper
|
||||
|
||||
|
||||
# Set on the spawn-child interpreter so the wrapper short-circuits when the
|
||||
# child resolves `module.qualname` back to its own decorated form, instead of
|
||||
# launching another subprocess.
|
||||
_SPAWN_CHILD_ENV = "VLLM_TEST_SPAWN_CHILD"
|
||||
|
||||
|
||||
def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]:
|
||||
"""Decorator to spawn a new process for each test function.
|
||||
|
||||
Uses subprocess with cloudpickle to serialize the test function and
|
||||
propagates exceptions back to the parent, so test failures are never
|
||||
silently swallowed (fixes https://github.com/vllm-project/vllm/issues/41415).
|
||||
Uses subprocess to run each test in a fresh interpreter and propagates
|
||||
exceptions back to the parent, so test failures are never silently
|
||||
swallowed (fixes https://github.com/vllm-project/vllm/issues/41415).
|
||||
|
||||
The child resolves the test function by importing its module and looking
|
||||
it up by qualified name, rather than reconstructing it from a cloudpickle
|
||||
blob. Pickling the function by value would also pickle its ``__globals__``
|
||||
by value — turning module-level singletons (e.g.
|
||||
``vllm.compilation.counter.compilation_counter``) into stale clones in
|
||||
the child, so increments performed by the production code in the child
|
||||
would never be observable to the test.
|
||||
"""
|
||||
|
||||
@functools.wraps(f)
|
||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
|
||||
if os.environ.get(_SPAWN_CHILD_ENV) == "1":
|
||||
return f(*args, **kwargs)
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".tb", mode="wb") as tmp:
|
||||
tb_file = tmp.name
|
||||
|
||||
try:
|
||||
# Serialize the function + args with cloudpickle so closures work
|
||||
payload = cloudpickle.dumps((f, args, kwargs, tb_file))
|
||||
payload = cloudpickle.dumps(
|
||||
{
|
||||
"module": f.__module__,
|
||||
"qualname": f.__qualname__,
|
||||
"args": args,
|
||||
"kwargs": kwargs,
|
||||
"tb_file": tb_file,
|
||||
}
|
||||
)
|
||||
|
||||
child_script = (
|
||||
"import sys, cloudpickle, traceback\n"
|
||||
"import sys, importlib, cloudpickle, traceback\n"
|
||||
"try:\n"
|
||||
" from _pytest.outcomes import Skipped\n"
|
||||
"except ImportError:\n"
|
||||
" class Skipped(BaseException): pass\n"
|
||||
"f, args, kwargs, tb_file = "
|
||||
"cloudpickle.loads(sys.stdin.buffer.read())\n"
|
||||
"data = cloudpickle.loads(sys.stdin.buffer.read())\n"
|
||||
"mod = importlib.import_module(data['module'])\n"
|
||||
"target = mod\n"
|
||||
"for name in data['qualname'].split('.'):\n"
|
||||
" target = getattr(target, name)\n"
|
||||
"try:\n"
|
||||
" f(*args, **kwargs)\n"
|
||||
" target(*data['args'], **data['kwargs'])\n"
|
||||
"except Skipped:\n"
|
||||
" sys.exit(0)\n"
|
||||
"except BaseException:\n"
|
||||
" open(tb_file, 'w').write(traceback.format_exc())\n"
|
||||
" with open(data['tb_file'], 'w') as fp:\n"
|
||||
" fp.write(traceback.format_exc())\n"
|
||||
" sys.exit(1)\n"
|
||||
)
|
||||
|
||||
repo_root = str(VLLM_PATH.resolve())
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")
|
||||
env[_SPAWN_CHILD_ENV] = "1"
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", child_script],
|
||||
@@ -1557,12 +1586,14 @@ def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
# Read traceback written by child, fall back to stderr
|
||||
tb = ""
|
||||
if os.path.exists(tb_file) and os.path.getsize(tb_file) > 0:
|
||||
# Prefer the child's traceback file; fall back to stderr if
|
||||
# the child crashed before its except handler ran.
|
||||
try:
|
||||
with open(tb_file) as fp:
|
||||
tb = fp.read()
|
||||
else:
|
||||
except OSError:
|
||||
tb = ""
|
||||
if not tb:
|
||||
tb = result.stderr.decode()
|
||||
raise RuntimeError(
|
||||
f"Test subprocess '{f.__name__}' failed "
|
||||
|
||||
Reference in New Issue
Block a user