[torch.compile] Enable attention and allreduce fusion without custom ops enabled (#24604)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
+101
-3
@@ -1,10 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# ruff: noqa
|
||||
import contextlib
|
||||
import pathlib
|
||||
from copy import deepcopy
|
||||
|
||||
from tblib import pickling_support
|
||||
|
||||
# ruff: noqa
|
||||
|
||||
# Install support for pickling exceptions so that we can nicely propagate
|
||||
# failures from tests running in a subprocess.
|
||||
# This should be run before any custom exception subclasses are defined.
|
||||
@@ -40,7 +43,7 @@ from transformers import (
|
||||
from transformers.models.auto.auto_factory import _BaseAutoModelClass
|
||||
|
||||
from tests.models.utils import TokensTextLogprobs, TokensTextLogprobsPromptLogprobs
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm import LLM, SamplingParams, envs
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset
|
||||
@@ -1070,6 +1073,101 @@ def caplog_vllm(temporary_enable_log_propagate, caplog):
|
||||
yield caplog
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def caplog_mp_fork():
|
||||
"""
|
||||
This fixture enables capturing logs from a forked MP subprocess.
|
||||
It should be used in conjunction with caplog_vllm.
|
||||
|
||||
By default, subprocess logs do not go through the parent process.
|
||||
We instead create a queue listener in the parent process which
|
||||
forwards logs to the logger's other handlers, and add a QueueHandler
|
||||
to the root logger. Forked subprocesses will inherit the root logger
|
||||
and pass their messages to the queue, which the listener will forward
|
||||
to the root logger, which can be captured by caplog.
|
||||
|
||||
Note that this workaround only works for fork; with spawn, the subprocess
|
||||
reinitializes logging and does not automatically inherit the queue.
|
||||
We'd have to manually pass the queue to the subprocess at the spawn point.
|
||||
See caplog_mp_spawn below.
|
||||
"""
|
||||
|
||||
@contextlib.contextmanager
|
||||
def ctx():
|
||||
import logging.handlers
|
||||
import multiprocessing as mp
|
||||
|
||||
logger_queue: mp.Queue[logging.LogRecord] = mp.Queue()
|
||||
logger = logging.getLogger()
|
||||
handlers = logger.handlers
|
||||
|
||||
# The listener works on a background thread, not inherited by the child.
|
||||
queue_listener = logging.handlers.QueueListener(logger_queue, *handlers)
|
||||
queue_listener.start()
|
||||
|
||||
# Add queue handler after creating the listener to avoid cycle
|
||||
logger.addHandler(logging.handlers.QueueHandler(logger_queue))
|
||||
yield
|
||||
queue_listener.stop()
|
||||
|
||||
return ctx
|
||||
|
||||
|
||||
class LogHolder:
|
||||
def __init__(self):
|
||||
self.text = None
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def caplog_mp_spawn(tmp_path, monkeypatch):
|
||||
"""
|
||||
This fixture enables capturing logs from a forked MP subprocess.
|
||||
It does not require caplog_vllm (but it only contains logs from the child).
|
||||
|
||||
By default, subprocess logs do not go through the parent process.
|
||||
We instead add a FileHandler to the config so the spawned child process
|
||||
writes its logs to a temp file.
|
||||
In the parent, we read the file and return the contents.
|
||||
|
||||
Note: this method could be extended to fork by either reconfiguring logging
|
||||
in the parent or using a SocketHandler:
|
||||
https://docs.python.org/3/howto/logging-cookbook.html#sending-and-receiving-logging-events-across-a-network # noqa: E501
|
||||
"""
|
||||
|
||||
@contextlib.contextmanager
|
||||
def ctx(level: int | str):
|
||||
from vllm.logger import DEFAULT_LOGGING_CONFIG
|
||||
|
||||
config_path = tmp_path / "vllm_logging_config.json"
|
||||
log_path = tmp_path / "vllm.log"
|
||||
log_holder = LogHolder()
|
||||
|
||||
config = deepcopy(DEFAULT_LOGGING_CONFIG)
|
||||
if envs.VLLM_LOGGING_CONFIG_PATH:
|
||||
path = pathlib.Path(envs.VLLM_LOGGING_CONFIG_PATH)
|
||||
assert path.exists()
|
||||
config = json.loads(path.read_text())
|
||||
|
||||
config["loggers"]["vllm"]["handlers"] += ["vllm_file"]
|
||||
config["handlers"]["vllm_file"] = {
|
||||
"class": "logging.FileHandler",
|
||||
"formatter": "vllm",
|
||||
"level": level,
|
||||
"filename": log_path.as_posix(),
|
||||
}
|
||||
|
||||
config_path.write_text(json.dumps(config))
|
||||
|
||||
with monkeypatch.context() as monkeypatch_ctx:
|
||||
monkeypatch_ctx.setenv("VLLM_LOGGING_CONFIG_PATH", config_path.as_posix())
|
||||
monkeypatch_ctx.setenv("VLLM_CONFIGURE_LOGGING", "1")
|
||||
yield log_holder
|
||||
|
||||
log_holder.text = log_path.read_text()
|
||||
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def num_gpus_available():
|
||||
"""Get number of GPUs without initializing the CUDA context
|
||||
|
||||
Reference in New Issue
Block a user