[Bugfix] Fix spawn_new_process_for_each_test silently swallowing test failures (#41423)
Signed-off-by: dqzhengAP <dqzheng1996@gmail.com>
This commit is contained in:
+52
-39
@@ -19,7 +19,7 @@ import threading
|
|||||||
import time
|
import time
|
||||||
import warnings
|
import warnings
|
||||||
from collections.abc import Callable, Iterable, Sequence
|
from collections.abc import Callable, Iterable, Sequence
|
||||||
from contextlib import ExitStack, contextmanager, suppress
|
from contextlib import ExitStack, contextmanager
|
||||||
from multiprocessing import Process
|
from multiprocessing import Process
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal
|
||||||
@@ -1512,52 +1512,65 @@ def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, Non
|
|||||||
|
|
||||||
|
|
||||||
def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]:
|
def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]:
|
||||||
"""Decorator to spawn a new process for each test function."""
|
"""Decorator to spawn a new process for each test function.
|
||||||
|
|
||||||
|
Uses subprocess with cloudpickle to serialize the test function and
|
||||||
|
propagates exceptions back to the parent, so test failures are never
|
||||||
|
silently swallowed (fixes https://github.com/vllm-project/vllm/issues/41415).
|
||||||
|
"""
|
||||||
|
|
||||||
@functools.wraps(f)
|
@functools.wraps(f)
|
||||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
|
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
|
||||||
# Check if we're already in a subprocess
|
with tempfile.NamedTemporaryFile(delete=False, suffix=".tb", mode="wb") as tmp:
|
||||||
if os.environ.get("RUNNING_IN_SUBPROCESS") == "1":
|
tb_file = tmp.name
|
||||||
# If we are, just run the function directly
|
|
||||||
return f(*args, **kwargs)
|
|
||||||
|
|
||||||
import torch.multiprocessing as mp
|
try:
|
||||||
|
# Serialize the function + args with cloudpickle so closures work
|
||||||
|
payload = cloudpickle.dumps((f, args, kwargs, tb_file))
|
||||||
|
|
||||||
with suppress(RuntimeError):
|
child_script = (
|
||||||
mp.set_start_method("spawn")
|
"import sys, cloudpickle, traceback\n"
|
||||||
|
"try:\n"
|
||||||
# Get the module
|
" from _pytest.outcomes import Skipped\n"
|
||||||
module_name = f.__module__
|
"except ImportError:\n"
|
||||||
|
" class Skipped(BaseException): pass\n"
|
||||||
# Create a process with environment variable set
|
"f, args, kwargs, tb_file = "
|
||||||
env = os.environ.copy()
|
"cloudpickle.loads(sys.stdin.buffer.read())\n"
|
||||||
env["RUNNING_IN_SUBPROCESS"] = "1"
|
"try:\n"
|
||||||
|
" f(*args, **kwargs)\n"
|
||||||
with tempfile.TemporaryDirectory() as tempdir:
|
"except Skipped:\n"
|
||||||
output_filepath = os.path.join(tempdir, "new_process.tmp")
|
" sys.exit(0)\n"
|
||||||
|
"except BaseException:\n"
|
||||||
# `cloudpickle` allows pickling complex functions directly
|
" open(tb_file, 'w').write(traceback.format_exc())\n"
|
||||||
input_bytes = cloudpickle.dumps((f, output_filepath))
|
" sys.exit(1)\n"
|
||||||
|
|
||||||
repo_root = str(VLLM_PATH.resolve())
|
|
||||||
|
|
||||||
env = dict(env or os.environ)
|
|
||||||
env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")
|
|
||||||
|
|
||||||
cmd = [sys.executable, "-m", f"{module_name}"]
|
|
||||||
|
|
||||||
returned = subprocess.run(
|
|
||||||
cmd, input=input_bytes, capture_output=True, env=env
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# check if the subprocess is successful
|
repo_root = str(VLLM_PATH.resolve())
|
||||||
try:
|
env = os.environ.copy()
|
||||||
returned.check_returncode()
|
env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")
|
||||||
except Exception as e:
|
|
||||||
# wrap raised exception to provide more information
|
result = subprocess.run(
|
||||||
|
[sys.executable, "-c", child_script],
|
||||||
|
input=payload,
|
||||||
|
capture_output=True,
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
# Read traceback written by child, fall back to stderr
|
||||||
|
tb = ""
|
||||||
|
if os.path.exists(tb_file) and os.path.getsize(tb_file) > 0:
|
||||||
|
with open(tb_file) as fp:
|
||||||
|
tb = fp.read()
|
||||||
|
else:
|
||||||
|
tb = result.stderr.decode()
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"Error raised in subprocess:\n{returned.stderr.decode()}"
|
f"Test subprocess '{f.__name__}' failed "
|
||||||
) from e
|
f"(exit code {result.returncode}):\n{tb}"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
with contextlib.suppress(OSError):
|
||||||
|
os.remove(tb_file)
|
||||||
|
|
||||||
return wrapper
|
return wrapper
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Tests for spawn_new_process_for_each_test decorator."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.utils import spawn_new_process_for_each_test
|
||||||
|
|
||||||
|
|
||||||
|
@spawn_new_process_for_each_test
|
||||||
|
def test_spawn_decorator_passing():
|
||||||
|
"""Passing function should complete normally."""
|
||||||
|
assert 1 + 1 == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.xfail(raises=RuntimeError, strict=True)
|
||||||
|
@spawn_new_process_for_each_test
|
||||||
|
def test_spawn_decorator_failure_is_caught():
|
||||||
|
"""Failing function should raise RuntimeError, never silently pass."""
|
||||||
|
raise ValueError("intentional failure")
|
||||||
|
|
||||||
|
|
||||||
|
@spawn_new_process_for_each_test
|
||||||
|
def test_spawn_decorator_skip():
|
||||||
|
"""pytest.skip inside subprocess should propagate correctly."""
|
||||||
|
pytest.skip("intentional skip")
|
||||||
|
|
||||||
|
|
||||||
|
@spawn_new_process_for_each_test
|
||||||
|
@pytest.mark.parametrize("x,y,expected", [(1, 2, 3), (0, 0, 0)])
|
||||||
|
def test_spawn_decorator_parametrized(x, y, expected):
|
||||||
|
"""Args and kwargs must be forwarded correctly to subprocess."""
|
||||||
|
assert x + y == expected
|
||||||
@@ -371,193 +371,200 @@ class TestCUDAGraphWrapper:
|
|||||||
assert not wrapper.concrete_cudagraph_entries
|
assert not wrapper.concrete_cudagraph_entries
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
def _run_and_monitor_call(
|
||||||
class TestCudagraphIntegration:
|
wrapper, input_tensor, runtime_mode, batch_descriptor, vllm_config
|
||||||
def setup_method(self):
|
):
|
||||||
# only FULL mode for non-uniform batches
|
"""Helper to run a single call and monitor the action."""
|
||||||
self.comp_config = CompilationConfig(
|
|
||||||
mode=CompilationMode.VLLM_COMPILE,
|
|
||||||
cudagraph_mode="FULL",
|
|
||||||
cudagraph_capture_sizes=[10, 20],
|
|
||||||
)
|
|
||||||
self.vllm_config = _create_vllm_config(self.comp_config)
|
|
||||||
self.dispatcher = CudagraphDispatcher(self.vllm_config)
|
|
||||||
self.dispatcher.initialize_cudagraph_keys(
|
|
||||||
self.comp_config.cudagraph_mode, uniform_decode_query_len=1
|
|
||||||
)
|
|
||||||
|
|
||||||
def _run_and_monitor_call(
|
with (
|
||||||
self, wrapper, input_tensor, runtime_mode, batch_descriptor
|
patch("torch.cuda.graph", wraps=torch.cuda.graph) as mock_graph_context,
|
||||||
|
patch.object(wrapper, "runnable", wraps=wrapper.runnable) as mock_runnable,
|
||||||
):
|
):
|
||||||
"""Helper to run a single call and monitor the action."""
|
entry = wrapper.concrete_cudagraph_entries.get(batch_descriptor, None)
|
||||||
|
|
||||||
with (
|
context = set_forward_context(
|
||||||
patch("torch.cuda.graph", wraps=torch.cuda.graph) as mock_graph_context,
|
|
||||||
patch.object(wrapper, "runnable", wraps=wrapper.runnable) as mock_runnable,
|
|
||||||
):
|
|
||||||
entry = wrapper.concrete_cudagraph_entries.get(batch_descriptor, None)
|
|
||||||
|
|
||||||
context = set_forward_context(
|
|
||||||
attn_metadata=None,
|
|
||||||
vllm_config=self.vllm_config,
|
|
||||||
cudagraph_runtime_mode=runtime_mode,
|
|
||||||
batch_descriptor=batch_descriptor,
|
|
||||||
)
|
|
||||||
mock_replay = MagicMock()
|
|
||||||
if entry and entry.cudagraph:
|
|
||||||
with (
|
|
||||||
context,
|
|
||||||
patch.object(
|
|
||||||
entry.cudagraph, "replay", new_callable=MagicMock
|
|
||||||
) as mock_replay,
|
|
||||||
):
|
|
||||||
wrapper(input_tensor)
|
|
||||||
else:
|
|
||||||
with context:
|
|
||||||
wrapper(input_tensor)
|
|
||||||
|
|
||||||
if mock_graph_context.called:
|
|
||||||
# note that this is globally mocked, so it will be detected
|
|
||||||
# even whether called by the inner or outer wrapper
|
|
||||||
return "capture_global"
|
|
||||||
if mock_replay.called:
|
|
||||||
# only for outer wrapper
|
|
||||||
return "replay"
|
|
||||||
if mock_runnable.call_count > 0:
|
|
||||||
# only for outer wrapper
|
|
||||||
return "bypass"
|
|
||||||
return "unknown"
|
|
||||||
|
|
||||||
@create_new_process_for_each_test("spawn")
|
|
||||||
def test_capture_replay_bypass_logic(self):
|
|
||||||
model = SimpleMLP().to(DEVICE_TYPE)
|
|
||||||
full_wrapper = CUDAGraphWrapper(model, self.vllm_config, CUDAGraphMode.FULL)
|
|
||||||
max_bs = 16
|
|
||||||
persistent_input_buffer = torch.zeros(max_bs, 10, device=DEVICE_TYPE)
|
|
||||||
input_1 = persistent_input_buffer[:1]
|
|
||||||
input_2 = persistent_input_buffer[:2]
|
|
||||||
input_3 = persistent_input_buffer[:3]
|
|
||||||
|
|
||||||
desc_1 = BatchDescriptor(num_tokens=1)
|
|
||||||
desc_2 = BatchDescriptor(num_tokens=2)
|
|
||||||
desc_3_unseen = BatchDescriptor(num_tokens=3)
|
|
||||||
|
|
||||||
# 0. global warmup
|
|
||||||
with set_forward_context(
|
|
||||||
attn_metadata=None,
|
attn_metadata=None,
|
||||||
vllm_config=self.vllm_config,
|
vllm_config=vllm_config,
|
||||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
cudagraph_runtime_mode=runtime_mode,
|
||||||
batch_descriptor=None,
|
batch_descriptor=batch_descriptor,
|
||||||
):
|
|
||||||
full_wrapper(input_1)
|
|
||||||
|
|
||||||
rt_mode, key = self.dispatcher.dispatch(num_tokens=desc_1.num_tokens)
|
|
||||||
# 1. Capture first shape
|
|
||||||
action = self._run_and_monitor_call(full_wrapper, input_1, rt_mode, key)
|
|
||||||
assert action == "capture_global"
|
|
||||||
|
|
||||||
# 2. Replay first shape
|
|
||||||
action = self._run_and_monitor_call(full_wrapper, input_1, rt_mode, key)
|
|
||||||
assert action == "replay"
|
|
||||||
|
|
||||||
rt_mode, key = self.dispatcher.dispatch(num_tokens=desc_2.num_tokens)
|
|
||||||
# 3. Capture second shape
|
|
||||||
action = self._run_and_monitor_call(full_wrapper, input_2, rt_mode, key)
|
|
||||||
assert action == "capture_global"
|
|
||||||
|
|
||||||
# 4. Replay second shape
|
|
||||||
action = self._run_and_monitor_call(
|
|
||||||
full_wrapper, input_2, CUDAGraphMode.FULL, desc_2
|
|
||||||
)
|
)
|
||||||
assert action == "replay"
|
mock_replay = MagicMock()
|
||||||
|
if entry and entry.cudagraph:
|
||||||
|
with (
|
||||||
|
context,
|
||||||
|
patch.object(
|
||||||
|
entry.cudagraph, "replay", new_callable=MagicMock
|
||||||
|
) as mock_replay,
|
||||||
|
):
|
||||||
|
wrapper(input_tensor)
|
||||||
|
else:
|
||||||
|
with context:
|
||||||
|
wrapper(input_tensor)
|
||||||
|
|
||||||
# 5. Bypass if no key match
|
if mock_graph_context.called:
|
||||||
rt_mode, key = self.dispatcher.dispatch(num_tokens=desc_3_unseen.num_tokens)
|
# note that this is globally mocked, so it will be detected
|
||||||
assert rt_mode == CUDAGraphMode.NONE
|
# even whether called by the inner or outer wrapper
|
||||||
action = self._run_and_monitor_call(full_wrapper, input_3, rt_mode, key)
|
return "capture_global"
|
||||||
assert action == "bypass"
|
if mock_replay.called:
|
||||||
|
# only for outer wrapper
|
||||||
|
return "replay"
|
||||||
|
if mock_runnable.call_count > 0:
|
||||||
|
# only for outer wrapper
|
||||||
|
return "bypass"
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
# capture unseen shape is not allowed after disable
|
|
||||||
set_cudagraph_capturing_enabled(False)
|
|
||||||
with pytest.raises(RuntimeError):
|
|
||||||
self._run_and_monitor_call(
|
|
||||||
full_wrapper, input_3, CUDAGraphMode.FULL, desc_3_unseen
|
|
||||||
)
|
|
||||||
set_cudagraph_capturing_enabled(True)
|
|
||||||
|
|
||||||
@create_new_process_for_each_test("spawn")
|
@create_new_process_for_each_test("spawn")
|
||||||
def test_nested_wrappers(self):
|
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||||
"""Tests a scenario with a PIECEWISE wrapper inside a FULL one."""
|
def test_capture_replay_bypass_logic():
|
||||||
model = SimpleMLP().to(DEVICE_TYPE)
|
comp_config = CompilationConfig(
|
||||||
full_wrapper = CUDAGraphWrapper(model, self.vllm_config, CUDAGraphMode.FULL)
|
mode=CompilationMode.VLLM_COMPILE,
|
||||||
input_1 = torch.randn(1, 10, device=DEVICE_TYPE)
|
cudagraph_mode="FULL",
|
||||||
|
cudagraph_capture_sizes=[1, 2],
|
||||||
|
)
|
||||||
|
vllm_config = _create_vllm_config(comp_config)
|
||||||
|
dispatcher = CudagraphDispatcher(vllm_config)
|
||||||
|
dispatcher.initialize_cudagraph_keys(
|
||||||
|
comp_config.cudagraph_mode, uniform_decode_query_len=1
|
||||||
|
)
|
||||||
|
model = SimpleMLP().to(DEVICE_TYPE)
|
||||||
|
full_wrapper = CUDAGraphWrapper(model, vllm_config, CUDAGraphMode.FULL)
|
||||||
|
max_bs = 16
|
||||||
|
persistent_input_buffer = torch.zeros(max_bs, 10, device=DEVICE_TYPE)
|
||||||
|
input_1 = persistent_input_buffer[:1]
|
||||||
|
input_2 = persistent_input_buffer[:2]
|
||||||
|
input_3 = persistent_input_buffer[:3]
|
||||||
|
|
||||||
# Setup: Inner model is wrapped with PIECEWISE, outer with FULL
|
desc_1 = BatchDescriptor(num_tokens=1)
|
||||||
inner_model = SimpleMLP().to(DEVICE_TYPE)
|
desc_2 = BatchDescriptor(num_tokens=2)
|
||||||
piecewise_wrapper = CUDAGraphWrapper(
|
desc_3_unseen = BatchDescriptor(num_tokens=3)
|
||||||
inner_model, self.vllm_config, CUDAGraphMode.PIECEWISE
|
|
||||||
)
|
# 0. global warmup
|
||||||
inner_model.forward = MagicMock(wraps=inner_model.forward)
|
with set_forward_context(
|
||||||
outer_model = SimpleMLP().to(DEVICE_TYPE)
|
attn_metadata=None,
|
||||||
# When outer model is called, it calls the piecewise_wrapper
|
vllm_config=vllm_config,
|
||||||
outer_model.forward = MagicMock(
|
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||||
wraps=outer_model.forward, side_effect=piecewise_wrapper
|
batch_descriptor=None,
|
||||||
)
|
):
|
||||||
full_wrapper = CUDAGraphWrapper(
|
full_wrapper(input_1)
|
||||||
outer_model, self.vllm_config, CUDAGraphMode.FULL
|
|
||||||
|
rt_mode, key = dispatcher.dispatch(num_tokens=desc_1.num_tokens)
|
||||||
|
# 1. Capture first shape
|
||||||
|
action = _run_and_monitor_call(full_wrapper, input_1, rt_mode, key, vllm_config)
|
||||||
|
assert action == "capture_global"
|
||||||
|
|
||||||
|
# 2. Replay first shape
|
||||||
|
action = _run_and_monitor_call(full_wrapper, input_1, rt_mode, key, vllm_config)
|
||||||
|
assert action == "replay"
|
||||||
|
|
||||||
|
rt_mode, key = dispatcher.dispatch(num_tokens=desc_2.num_tokens)
|
||||||
|
# 3. Capture second shape
|
||||||
|
action = _run_and_monitor_call(full_wrapper, input_2, rt_mode, key, vllm_config)
|
||||||
|
assert action == "capture_global"
|
||||||
|
|
||||||
|
# 4. Replay second shape
|
||||||
|
action = _run_and_monitor_call(
|
||||||
|
full_wrapper, input_2, CUDAGraphMode.FULL, key, vllm_config
|
||||||
|
)
|
||||||
|
assert action == "replay"
|
||||||
|
|
||||||
|
# 5. Bypass if no key match
|
||||||
|
rt_mode, key = dispatcher.dispatch(num_tokens=desc_3_unseen.num_tokens)
|
||||||
|
assert rt_mode == CUDAGraphMode.NONE
|
||||||
|
action = _run_and_monitor_call(full_wrapper, input_3, rt_mode, key, vllm_config)
|
||||||
|
assert action == "bypass"
|
||||||
|
|
||||||
|
# capture unseen shape is not allowed after disable
|
||||||
|
set_cudagraph_capturing_enabled(False)
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
_run_and_monitor_call(
|
||||||
|
full_wrapper, input_3, CUDAGraphMode.FULL, desc_3_unseen, vllm_config
|
||||||
)
|
)
|
||||||
|
set_cudagraph_capturing_enabled(True)
|
||||||
|
|
||||||
desc_1 = BatchDescriptor(num_tokens=1)
|
|
||||||
|
|
||||||
# 0. global warmup
|
@create_new_process_for_each_test("spawn")
|
||||||
with set_forward_context(
|
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||||
attn_metadata=None,
|
def test_nested_wrappers():
|
||||||
vllm_config=self.vllm_config,
|
"""Tests a scenario with a PIECEWISE wrapper inside a FULL one."""
|
||||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
comp_config = CompilationConfig(
|
||||||
batch_descriptor=None,
|
mode=CompilationMode.VLLM_COMPILE,
|
||||||
):
|
cudagraph_mode="FULL",
|
||||||
full_wrapper(input_1)
|
cudagraph_capture_sizes=[1],
|
||||||
|
)
|
||||||
|
vllm_config = _create_vllm_config(comp_config)
|
||||||
|
dispatcher = CudagraphDispatcher(vllm_config)
|
||||||
|
dispatcher.initialize_cudagraph_keys(
|
||||||
|
comp_config.cudagraph_mode, uniform_decode_query_len=1
|
||||||
|
)
|
||||||
|
model = SimpleMLP().to(DEVICE_TYPE)
|
||||||
|
full_wrapper = CUDAGraphWrapper(model, vllm_config, CUDAGraphMode.FULL)
|
||||||
|
input_1 = torch.randn(1, 10, device=DEVICE_TYPE)
|
||||||
|
|
||||||
# --- Test runtime mode FULL---
|
# Setup: Inner model is wrapped with PIECEWISE, outer with FULL
|
||||||
# Run with FULL mode context. Expect outer wrapper to capture.
|
inner_model = SimpleMLP().to(DEVICE_TYPE)
|
||||||
# The inner mock should be called once inside the graph capture.
|
piecewise_wrapper = CUDAGraphWrapper(
|
||||||
outer_model.forward.reset_mock()
|
inner_model, vllm_config, CUDAGraphMode.PIECEWISE
|
||||||
inner_model.forward.reset_mock()
|
)
|
||||||
action = self._run_and_monitor_call(
|
inner_model.forward = MagicMock(wraps=inner_model.forward)
|
||||||
full_wrapper, input_1, CUDAGraphMode.FULL, desc_1
|
outer_model = SimpleMLP().to(DEVICE_TYPE)
|
||||||
)
|
# When outer model is called, it calls the piecewise_wrapper
|
||||||
assert action == "capture_global"
|
outer_model.forward = MagicMock(
|
||||||
assert outer_model.forward.call_count == 1
|
wraps=outer_model.forward, side_effect=piecewise_wrapper
|
||||||
assert inner_model.forward.call_count == 1
|
)
|
||||||
|
full_wrapper = CUDAGraphWrapper(outer_model, vllm_config, CUDAGraphMode.FULL)
|
||||||
|
|
||||||
# Run again. Expect outer wrapper to replay.
|
desc_1 = BatchDescriptor(num_tokens=1)
|
||||||
# The outer model should NOT be called because the whole graph
|
|
||||||
# is replayed.
|
|
||||||
action = self._run_and_monitor_call(
|
|
||||||
full_wrapper, input_1, CUDAGraphMode.FULL, desc_1
|
|
||||||
)
|
|
||||||
assert action == "replay"
|
|
||||||
assert outer_model.forward.call_count == 1 # No new call
|
|
||||||
assert inner_model.forward.call_count == 1
|
|
||||||
|
|
||||||
# --- Test runtime mode PIECEWISE ---
|
# 0. global warmup
|
||||||
outer_model.forward.reset_mock()
|
with set_forward_context(
|
||||||
inner_model.forward.reset_mock()
|
attn_metadata=None,
|
||||||
# Run with PIECEWISE mode context.
|
vllm_config=vllm_config,
|
||||||
# Expect outer wrapper to bypass and call inner wrapper.
|
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||||
# Inner wrapper should capture.
|
batch_descriptor=None,
|
||||||
action = self._run_and_monitor_call(
|
):
|
||||||
full_wrapper, input_1, CUDAGraphMode.PIECEWISE, desc_1
|
full_wrapper(input_1)
|
||||||
)
|
|
||||||
assert action == "capture_global"
|
|
||||||
assert outer_model.forward.call_count == 1
|
|
||||||
assert inner_model.forward.call_count == 1
|
|
||||||
|
|
||||||
# Run again with PIECEWISE.
|
# --- Test runtime mode FULL---
|
||||||
# Outer bypasses, inner replays.
|
# Run with FULL mode context. Expect outer wrapper to capture.
|
||||||
action = self._run_and_monitor_call(
|
# The inner mock should be called once inside the graph capture.
|
||||||
full_wrapper, input_1, CUDAGraphMode.PIECEWISE, desc_1
|
outer_model.forward.reset_mock()
|
||||||
)
|
inner_model.forward.reset_mock()
|
||||||
assert action == "bypass"
|
action = _run_and_monitor_call(
|
||||||
assert outer_model.forward.call_count == 2
|
full_wrapper, input_1, CUDAGraphMode.FULL, desc_1, vllm_config
|
||||||
assert inner_model.forward.call_count == 1
|
)
|
||||||
|
assert action == "capture_global"
|
||||||
|
assert outer_model.forward.call_count == 1
|
||||||
|
assert inner_model.forward.call_count == 1
|
||||||
|
|
||||||
|
# Run again. Expect outer wrapper to replay.
|
||||||
|
# The outer model should NOT be called because the whole graph
|
||||||
|
# is replayed.
|
||||||
|
action = _run_and_monitor_call(
|
||||||
|
full_wrapper, input_1, CUDAGraphMode.FULL, desc_1, vllm_config
|
||||||
|
)
|
||||||
|
assert action == "replay"
|
||||||
|
assert outer_model.forward.call_count == 1 # No new call
|
||||||
|
assert inner_model.forward.call_count == 1
|
||||||
|
|
||||||
|
# --- Test runtime mode PIECEWISE ---
|
||||||
|
outer_model.forward.reset_mock()
|
||||||
|
inner_model.forward.reset_mock()
|
||||||
|
# Run with PIECEWISE mode context.
|
||||||
|
# Expect outer wrapper to bypass and call inner wrapper.
|
||||||
|
# Inner wrapper should capture.
|
||||||
|
action = _run_and_monitor_call(
|
||||||
|
full_wrapper, input_1, CUDAGraphMode.PIECEWISE, desc_1, vllm_config
|
||||||
|
)
|
||||||
|
assert action == "capture_global"
|
||||||
|
assert outer_model.forward.call_count == 1
|
||||||
|
assert inner_model.forward.call_count == 1
|
||||||
|
|
||||||
|
# Run again with PIECEWISE.
|
||||||
|
# Outer bypasses, inner replays.
|
||||||
|
action = _run_and_monitor_call(
|
||||||
|
full_wrapper, input_1, CUDAGraphMode.PIECEWISE, desc_1, vllm_config
|
||||||
|
)
|
||||||
|
assert action == "bypass"
|
||||||
|
assert outer_model.forward.call_count == 2
|
||||||
|
assert inner_model.forward.call_count == 1
|
||||||
|
|||||||
@@ -120,12 +120,11 @@ api_keyword_args = {
|
|||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test()
|
@create_new_process_for_each_test()
|
||||||
@pytest.mark.asyncio
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"model_name",
|
"model_name",
|
||||||
[MODEL_NAME],
|
[MODEL_NAME],
|
||||||
)
|
)
|
||||||
async def test_custom_logitsprocs(client: openai.AsyncOpenAI, model_name: str):
|
def test_custom_logitsprocs(server, model_name: str):
|
||||||
"""Test custom logitsprocs when starting OpenAI server from CLI
|
"""Test custom logitsprocs when starting OpenAI server from CLI
|
||||||
|
|
||||||
Launch vLLM OpenAI-compatible server, configured to load a custom logitproc
|
Launch vLLM OpenAI-compatible server, configured to load a custom logitproc
|
||||||
@@ -139,36 +138,45 @@ async def test_custom_logitsprocs(client: openai.AsyncOpenAI, model_name: str):
|
|||||||
token
|
token
|
||||||
"""
|
"""
|
||||||
|
|
||||||
use_dummy_logitproc = True
|
import asyncio
|
||||||
for prompt in prompts:
|
|
||||||
# Build request arguments
|
async def _async_main(srv, mn):
|
||||||
request_keyword_args: dict[str, Any] = {
|
async with srv.get_async_client() as client:
|
||||||
**api_keyword_args,
|
await _run(client)
|
||||||
}
|
|
||||||
if use_dummy_logitproc:
|
async def _run(client):
|
||||||
# 50% of requests pass target_token custom arg
|
use_dummy_logitproc = True
|
||||||
target_token = random.choice([128, 67])
|
for prompt in prompts:
|
||||||
# For requests which activate the dummy logitproc, choose one of
|
# Build request arguments
|
||||||
# two `target_token` values which are known not to be EOS tokens
|
request_keyword_args: dict[str, Any] = {
|
||||||
request_keyword_args["extra_body"] = {
|
**api_keyword_args,
|
||||||
"vllm_xargs": {DUMMY_LOGITPROC_ARG: target_token}
|
|
||||||
}
|
}
|
||||||
batch = await client.completions.create(
|
if use_dummy_logitproc:
|
||||||
model=model_name,
|
# 50% of requests pass target_token custom arg
|
||||||
prompt=prompt,
|
target_token = random.choice([128, 67])
|
||||||
**request_keyword_args,
|
# For requests which activate the dummy logitproc, choose one of
|
||||||
)
|
# two `target_token` values which are known not to be EOS tokens
|
||||||
|
request_keyword_args["extra_body"] = {
|
||||||
|
"vllm_xargs": {DUMMY_LOGITPROC_ARG: target_token}
|
||||||
|
}
|
||||||
|
batch = await client.completions.create(
|
||||||
|
model=model_name,
|
||||||
|
prompt=prompt,
|
||||||
|
**request_keyword_args,
|
||||||
|
)
|
||||||
|
|
||||||
if use_dummy_logitproc:
|
if use_dummy_logitproc:
|
||||||
# Only for requests which activate dummy logitproc - validate that
|
# Only for requests which activate dummy logitproc - validate that
|
||||||
# output token is repeated
|
# output token is repeated
|
||||||
choices: openai.types.CompletionChoice = batch.choices
|
choices: openai.types.CompletionChoice = batch.choices
|
||||||
toks = choices[0].logprobs.tokens
|
toks = choices[0].logprobs.tokens
|
||||||
if not all([x == toks[0] for x in toks]):
|
if not all([x == toks[0] for x in toks]):
|
||||||
raise AssertionError(f"Generated {toks} should all be {toks[0]}")
|
raise AssertionError(f"Generated {toks} should all be {toks[0]}")
|
||||||
|
|
||||||
# Alternate whether to activate dummy logitproc for each request
|
# Alternate whether to activate dummy logitproc for each request
|
||||||
use_dummy_logitproc = not use_dummy_logitproc
|
use_dummy_logitproc = not use_dummy_logitproc
|
||||||
|
|
||||||
|
asyncio.run(_async_main(server, model_name))
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
Reference in New Issue
Block a user