[Feature] Warm up readonly multimodal processor during renderer startup (#40797)

Signed-off-by: Chenguang ZHENG <645327136@qq.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Chenguang Zheng
2026-04-25 03:57:41 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 428b988c98
commit 07351e0883
7 changed files with 368 additions and 27 deletions
@@ -164,6 +164,58 @@ async def test_chat_error_non_stream():
await serving_chat.create_chat_completion(request)
@pytest.mark.asyncio
async def test_openai_chat_keeps_mm_cache_for_engine_execution():
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_chat = _build_serving_chat(mock_engine)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
)
result = await serving_chat.render_chat_request(request)
assert isinstance(result, tuple)
assert (
serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
"skip_mm_cache"
]
is False
)
@pytest.mark.asyncio
async def test_renderer_only_chat_request_skips_mm_cache():
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_chat = _build_serving_chat(mock_engine)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
)
result = await serving_chat.openai_serving_render.render_chat_request(request)
assert result.token_ids == [1, 2, 3]
assert (
serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
"skip_mm_cache"
]
is True
)
@pytest.mark.asyncio
async def test_chat_error_stream():
"""test finish_reason='error' returns 500 InternalServerError (streaming)"""
@@ -3,7 +3,7 @@
from dataclasses import dataclass, field
from typing import Any
from unittest.mock import MagicMock
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -148,6 +148,66 @@ async def test_completion_error_non_stream():
await serving_completion.create_completion(request)
@pytest.mark.asyncio
async def test_openai_completion_keeps_mm_cache_for_engine_execution():
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_completion = _build_serving_completion(mock_engine)
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
request = CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
)
result = await serving_completion.render_completion_request(request)
assert isinstance(result, list)
assert (
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
is False
)
@pytest.mark.asyncio
async def test_renderer_only_completion_request_skips_mm_cache():
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_completion = _build_serving_completion(mock_engine)
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
request = CompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
)
result = await serving_completion.openai_serving_render.render_completion_request(
request
)
assert isinstance(result, list)
assert (
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
is True
)
@pytest.mark.asyncio
async def test_completion_error_stream():
"""test finish_reason='error' returns 500 InternalServerError (streaming)"""
@@ -12,7 +12,10 @@ from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.engine.protocol import StreamOptions
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
from vllm.entrypoints.serve.disagg.protocol import (
GenerateRequest,
GenerateResponse,
)
from vllm.entrypoints.serve.disagg.serving import ServingTokens
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.logprobs import Logprob
@@ -164,6 +167,36 @@ def _parse_sse_chunks(chunks: list[str]) -> list[Any]:
return parsed
@pytest.mark.asyncio
async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
stream=False,
)
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert (
serving.openai_serving_render.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
is True
)
@pytest.mark.asyncio
async def test_stream_basic():
"""Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""
@@ -0,0 +1,140 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.tokenize.protocol import (
TokenizeChatRequest,
TokenizeCompletionRequest,
)
from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
from vllm.v1.engine.async_llm import AsyncLLM
MODEL_NAME = "openai-community/gpt2"
BASE_MODEL_PATHS = [
BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
]
@dataclass
class MockHFConfig:
model_type: str = "any"
@dataclass
class MockModelConfig:
task = "generate"
runner_type = "generate"
model = MODEL_NAME
tokenizer = MODEL_NAME
trust_remote_code = False
tokenizer_mode = "auto"
max_model_len = 100
tokenizer_revision = None
multimodal_config = MultiModalConfig()
hf_config = MockHFConfig()
hf_text_config = MockHFConfig()
logits_processors: list[str] | None = None
diff_sampling_param: dict | None = None
allowed_local_media_path: str = ""
allowed_media_domains: list[str] | None = None
encoder_config = None
generation_config: str = "auto"
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
skip_tokenizer_init = False
is_encoder_decoder: bool = False
is_multimodal_model: bool = False
renderer_num_workers: int = 1
def get_diff_sampling_param(self):
return self.diff_sampling_param or {}
def _build_serving_tokenization(engine: AsyncLLM) -> OpenAIServingTokenization:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
return OpenAIServingTokenization(
engine,
models,
openai_serving_render=serving_render,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
@pytest.mark.asyncio
async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = MagicMock()
serving = _build_serving_tokenization(mock_engine)
serving.openai_serving_render.preprocess_chat = AsyncMock(
return_value=(
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
)
)
request = TokenizeChatRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
)
response = await serving.create_tokenize(request, MagicMock(headers={}))
assert response.tokens == [1, 2, 3]
assert (
serving.openai_serving_render.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is True
)
@pytest.mark.asyncio
async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
mock_engine = MagicMock(spec=AsyncLLM)
mock_engine.errored = False
mock_engine.model_config = MockModelConfig()
mock_engine.input_processor = MagicMock()
mock_engine.renderer = MagicMock()
serving = _build_serving_tokenization(mock_engine)
serving.openai_serving_render.preprocess_completion = AsyncMock(
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
request = TokenizeCompletionRequest(
model=MODEL_NAME,
prompt="Test prompt",
)
response = await serving.create_tokenize(request, MagicMock(headers={}))
assert response.tokens == [1, 2, 3]
assert (
serving.openai_serving_render.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
is True
)