[CI][Bugfix] Fix failure CI step "PyTorch Fullgraph Smoke Test" (#41953)

Signed-off-by: haosdent <haosdent@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
haosdent
2026-05-07 19:41:56 -07:00
committed by GitHub
co-authored by Roger Wang
parent 50f2db2555
commit 5f6a02812a
2 changed files with 53 additions and 15 deletions
+8 -1
View File
@@ -161,7 +161,14 @@ def _run_simple_model(
@pytest.mark.parametrize("intermediate_unbacked", [True, False])
@torch.inference_mode()
@create_new_process_for_each_test("spawn")
def test_simple_piecewise_compile(backend, intermediate_unbacked):
def test_simple_piecewise_compile(backend, intermediate_unbacked, monkeypatch):
# `intermediate_unbacked` flips a control-flow branch inside
# `SillyModel.forward`, but the AOT-compile cache key only hashes the
# forward function's qualname + line number, so both parametrize variants
# share the same cache slot. Disabling the cache forces each variant to
# compile fresh; otherwise the second-running variant loads the first's
# artifact and segfaults with an illegal memory access.
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
_run_simple_model(
splitting_ops=["silly::attention"],
use_inductor_graph_partition=False,
+45 -14
View File
@@ -1511,43 +1511,72 @@ def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, Non
return wrapper
# Set on the spawn-child interpreter so the wrapper short-circuits when the
# child resolves `module.qualname` back to its own decorated form, instead of
# launching another subprocess.
_SPAWN_CHILD_ENV = "VLLM_TEST_SPAWN_CHILD"
def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]:
"""Decorator to spawn a new process for each test function.
Uses subprocess with cloudpickle to serialize the test function and
propagates exceptions back to the parent, so test failures are never
silently swallowed (fixes https://github.com/vllm-project/vllm/issues/41415).
Uses subprocess to run each test in a fresh interpreter and propagates
exceptions back to the parent, so test failures are never silently
swallowed (fixes https://github.com/vllm-project/vllm/issues/41415).
The child resolves the test function by importing its module and looking
it up by qualified name, rather than reconstructing it from a cloudpickle
blob. Pickling the function by value would also pickle its ``__globals__``
by value — turning module-level singletons (e.g.
``vllm.compilation.counter.compilation_counter``) into stale clones in
the child, so increments performed by the production code in the child
would never be observable to the test.
"""
@functools.wraps(f)
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
if os.environ.get(_SPAWN_CHILD_ENV) == "1":
return f(*args, **kwargs)
with tempfile.NamedTemporaryFile(delete=False, suffix=".tb", mode="wb") as tmp:
tb_file = tmp.name
try:
# Serialize the function + args with cloudpickle so closures work
payload = cloudpickle.dumps((f, args, kwargs, tb_file))
payload = cloudpickle.dumps(
{
"module": f.__module__,
"qualname": f.__qualname__,
"args": args,
"kwargs": kwargs,
"tb_file": tb_file,
}
)
child_script = (
"import sys, cloudpickle, traceback\n"
"import sys, importlib, cloudpickle, traceback\n"
"try:\n"
" from _pytest.outcomes import Skipped\n"
"except ImportError:\n"
" class Skipped(BaseException): pass\n"
"f, args, kwargs, tb_file = "
"cloudpickle.loads(sys.stdin.buffer.read())\n"
"data = cloudpickle.loads(sys.stdin.buffer.read())\n"
"mod = importlib.import_module(data['module'])\n"
"target = mod\n"
"for name in data['qualname'].split('.'):\n"
" target = getattr(target, name)\n"
"try:\n"
" f(*args, **kwargs)\n"
" target(*data['args'], **data['kwargs'])\n"
"except Skipped:\n"
" sys.exit(0)\n"
"except BaseException:\n"
" open(tb_file, 'w').write(traceback.format_exc())\n"
" with open(data['tb_file'], 'w') as fp:\n"
" fp.write(traceback.format_exc())\n"
" sys.exit(1)\n"
)
repo_root = str(VLLM_PATH.resolve())
env = os.environ.copy()
env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")
env[_SPAWN_CHILD_ENV] = "1"
result = subprocess.run(
[sys.executable, "-c", child_script],
@@ -1557,12 +1586,14 @@ def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]
)
if result.returncode != 0:
# Read traceback written by child, fall back to stderr
tb = ""
if os.path.exists(tb_file) and os.path.getsize(tb_file) > 0:
# Prefer the child's traceback file; fall back to stderr if
# the child crashed before its except handler ran.
try:
with open(tb_file) as fp:
tb = fp.read()
else:
except OSError:
tb = ""
if not tb:
tb = result.stderr.decode()
raise RuntimeError(
f"Test subprocess '{f.__name__}' failed "