[Feature] Warm up readonly multimodal processor during renderer startup (#40797)
Signed-off-by: Chenguang ZHENG <645327136@qq.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
428b988c98
commit
07351e0883
@@ -3,7 +3,7 @@
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -148,6 +148,66 @@ async def test_completion_error_non_stream():
|
||||
await serving_completion.create_completion(request)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_completion_keeps_mm_cache_for_engine_execution():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_completion = _build_serving_completion(mock_engine)
|
||||
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
)
|
||||
|
||||
result = await serving_completion.render_completion_request(request)
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert (
|
||||
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_renderer_only_completion_request_skips_mm_cache():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
mock_engine.errored = False
|
||||
mock_engine.model_config = MockModelConfig()
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_completion = _build_serving_completion(mock_engine)
|
||||
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
prompt="Test prompt",
|
||||
)
|
||||
|
||||
result = await serving_completion.openai_serving_render.render_completion_request(
|
||||
request
|
||||
)
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert (
|
||||
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_error_stream():
|
||||
"""test finish_reason='error' returns 500 InternalServerError (streaming)"""
|
||||
|
||||
Reference in New Issue
Block a user