[Frontend] Split ServingRender into renderer and entrypoint. (#44285)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -17,9 +17,11 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
@@ -88,19 +90,19 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
serving_chat = OpenAIServingChat(
|
||||
engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=serving_render,
|
||||
online_renderer=online_renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -113,7 +115,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
|
||||
serving_chat.openai_serving_render.preprocess_chat = AsyncMock(
|
||||
serving_chat.online_renderer.preprocess_chat = AsyncMock(
|
||||
side_effect=_fake_preprocess_chat
|
||||
)
|
||||
return serving_chat
|
||||
@@ -187,13 +189,46 @@ async def test_openai_chat_keeps_mm_cache_for_engine_execution():
|
||||
|
||||
assert isinstance(result, tuple)
|
||||
assert (
|
||||
serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
serving_chat.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_render(engine: AsyncLLM) -> ServingRender:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
online_derenderer = OnlineDerenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
serving_render = ServingRender(models, online_renderer, online_derenderer)
|
||||
|
||||
async def _fake_preprocess_chat(*args, **kwargs):
|
||||
# return conversation, engine_inputs
|
||||
return (
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
|
||||
serving_render.online_renderer.preprocess_chat = AsyncMock(
|
||||
side_effect=_fake_preprocess_chat
|
||||
)
|
||||
return serving_render
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_renderer_only_chat_request_skips_mm_cache():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
@@ -202,20 +237,18 @@ async def test_renderer_only_chat_request_skips_mm_cache():
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
serving_render = _build_serving_render(mock_engine)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
)
|
||||
|
||||
result = await serving_chat.openai_serving_render.render_chat_request(request)
|
||||
result = await serving_render.render_chat_request(request)
|
||||
|
||||
assert result.token_ids == [1, 2, 3]
|
||||
assert (
|
||||
serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
serving_render.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@@ -38,7 +38,6 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
@@ -46,6 +45,7 @@ from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.parser import HarmonyParser
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.mistral import MistralRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
@@ -567,13 +567,12 @@ def _build_renderer(model_config: MockModelConfig):
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_render(
|
||||
def _build_online_renderer(
|
||||
engine, model_registry: OpenAIModelRegistry
|
||||
) -> OpenAIServingRender:
|
||||
return OpenAIServingRender(
|
||||
) -> OnlineRenderer:
|
||||
return OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=model_registry,
|
||||
request_logger=None,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
@@ -591,13 +590,13 @@ def _build_serving_chat(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
openai_serving_render = _build_serving_render(engine, models.registry)
|
||||
online_renderer = _build_online_renderer(engine, models.registry)
|
||||
|
||||
serving_chat = OpenAIServingChat(
|
||||
engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=openai_serving_render,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
@@ -621,13 +620,13 @@ async def _async_serving_chat_init():
|
||||
engine = MockEngine()
|
||||
|
||||
models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
|
||||
openai_serving_render = _build_serving_render(engine, models.registry)
|
||||
online_renderer = _build_online_renderer(engine, models.registry)
|
||||
|
||||
serving_completion = OpenAIServingChat(
|
||||
engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=openai_serving_render,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
@@ -1396,9 +1395,7 @@ class TestServingChatWithHarmony:
|
||||
messages=messages,
|
||||
include_reasoning=include_reasoning,
|
||||
)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
[
|
||||
@@ -1429,8 +1426,8 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the second turn's input
|
||||
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
input_messages_2, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
|
||||
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_2
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_2,
|
||||
@@ -1452,9 +1449,7 @@ class TestServingChatWithHarmony:
|
||||
{"role": "user", "content": "Hello"},
|
||||
]
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
[
|
||||
@@ -1478,9 +1473,7 @@ class TestServingChatWithHarmony:
|
||||
req = ChatCompletionRequest(
|
||||
model=MODEL_NAME, messages=messages, tools=weather_tools
|
||||
)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
[
|
||||
@@ -1503,9 +1496,7 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the first turn's input
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
[
|
||||
@@ -1549,8 +1540,8 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the second turn's input
|
||||
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages_2, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
|
||||
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_2
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_2,
|
||||
@@ -1588,9 +1579,7 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the first turn's input
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
[
|
||||
@@ -1634,8 +1623,8 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the second turn's input
|
||||
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages_2, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
|
||||
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_2
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_2,
|
||||
@@ -1686,8 +1675,8 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the third turn's input
|
||||
req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages_3, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req_3)
|
||||
input_messages_3, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_3
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_3,
|
||||
@@ -1751,8 +1740,8 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the fourth turn's input
|
||||
req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages_4, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req_4)
|
||||
input_messages_4, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_4
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_4,
|
||||
@@ -1802,9 +1791,7 @@ class TestServingChatWithHarmony:
|
||||
},
|
||||
]
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
@@ -1835,9 +1822,7 @@ class TestServingChatWithHarmony:
|
||||
},
|
||||
]
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
@@ -1866,9 +1851,7 @@ class TestServingChatWithHarmony:
|
||||
},
|
||||
]
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
@@ -1898,14 +1881,14 @@ async def test_tool_choice_validation_without_parser():
|
||||
engine_client=mock_engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
openai_serving_render = _build_serving_render(mock_engine, models.registry)
|
||||
online_renderer = _build_online_renderer(mock_engine, models.registry)
|
||||
|
||||
# Create serving_chat without tool_parser (enable_auto_tools=False)
|
||||
serving_chat = OpenAIServingChat(
|
||||
mock_engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=openai_serving_render,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
@@ -1967,13 +1950,13 @@ async def test_streaming_n_gt1_independent_tool_parsers():
|
||||
engine_client=mock_engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
openai_serving_render = _build_serving_render(mock_engine, models.registry)
|
||||
online_renderer = _build_online_renderer(mock_engine, models.registry)
|
||||
|
||||
serving_chat = OpenAIServingChat(
|
||||
mock_engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=openai_serving_render,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
|
||||
@@ -14,9 +14,11 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
@@ -77,10 +79,9 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -88,7 +89,7 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
|
||||
return OpenAIServingCompletion(
|
||||
engine,
|
||||
models,
|
||||
openai_serving_render=serving_render,
|
||||
online_renderer=online_renderer,
|
||||
request_logger=None,
|
||||
)
|
||||
|
||||
@@ -158,7 +159,7 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_completion = _build_serving_completion(mock_engine)
|
||||
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
|
||||
serving_completion.online_renderer.preprocess_completion = AsyncMock(
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
@@ -171,13 +172,48 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert (
|
||||
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
|
||||
serving_completion.online_renderer.preprocess_completion.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_render(engine: AsyncLLM) -> ServingRender:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
online_derenderer = OnlineDerenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
serving_render = ServingRender(models, online_renderer, online_derenderer)
|
||||
|
||||
async def _fake_preprocess_chat(*args, **kwargs):
|
||||
# return conversation, engine_inputs
|
||||
return (
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
|
||||
serving_render.online_renderer.preprocess_chat = AsyncMock(
|
||||
side_effect=_fake_preprocess_chat
|
||||
)
|
||||
return serving_render
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_renderer_only_completion_request_skips_mm_cache():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
@@ -186,8 +222,9 @@ async def test_renderer_only_completion_request_skips_mm_cache():
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_completion = _build_serving_completion(mock_engine)
|
||||
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
|
||||
serving_render = _build_serving_render(mock_engine)
|
||||
|
||||
serving_render.online_renderer.preprocess_completion = AsyncMock(
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
@@ -196,13 +233,11 @@ async def test_renderer_only_completion_request_skips_mm_cache():
|
||||
prompt="Test prompt",
|
||||
)
|
||||
|
||||
result = await serving_completion.openai_serving_render.render_completion_request(
|
||||
request
|
||||
)
|
||||
result = await serving_render.render_completion_request(request)
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert (
|
||||
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
|
||||
serving_render.online_renderer.preprocess_completion.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
is True
|
||||
|
||||
@@ -14,10 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
@@ -144,16 +144,15 @@ def mock_serving_setup():
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=mock_engine.model_config,
|
||||
renderer=mock_engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
serving_completion = OpenAIServingCompletion(
|
||||
mock_engine, models, openai_serving_render=serving_render, request_logger=None
|
||||
mock_engine, models, online_renderer=online_renderer, request_logger=None
|
||||
)
|
||||
|
||||
return mock_engine, serving_completion
|
||||
|
||||
@@ -230,7 +230,7 @@ class TestInitializeToolSessions:
|
||||
instance = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
openai_serving_render=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -316,7 +316,7 @@ class TestValidateGeneratorInput:
|
||||
instance = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
openai_serving_render=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -379,7 +379,7 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
serving = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
openai_serving_render=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -684,7 +684,7 @@ def _make_serving_instance_with_reasoning():
|
||||
serving = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
openai_serving_render=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
|
||||
@@ -428,8 +428,8 @@ class TestPreProcessCohereOnline:
|
||||
handler._get_task_instruction_prefix = lambda _input_type: None
|
||||
handler._has_chat_template = lambda: False
|
||||
handler._preprocess_cmpl_online = preprocess_cmpl_online
|
||||
handler._batch_render_chat = lambda *_args, **_kwargs: (
|
||||
pytest.fail("text-only request should not require chat rendering")
|
||||
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
|
||||
"text-only request should not require chat rendering"
|
||||
)
|
||||
|
||||
handler._pre_process_cohere_online(ctx)
|
||||
@@ -448,8 +448,8 @@ class TestPreProcessCohereOnline:
|
||||
|
||||
handler._get_task_instruction_prefix = lambda _input_type: "query: "
|
||||
handler._has_chat_template = lambda: False
|
||||
handler._batch_render_chat = lambda *_args, **_kwargs: (
|
||||
pytest.fail("chat rendering should be skipped without a template")
|
||||
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
|
||||
"chat rendering should be skipped without a template"
|
||||
)
|
||||
handler._preprocess_cmpl_online = preprocess_cmpl
|
||||
|
||||
@@ -485,8 +485,8 @@ class TestPreProcessCohereOnline:
|
||||
handler._get_task_instruction_prefix = lambda _input_type: "query: "
|
||||
handler._has_chat_template = lambda: True
|
||||
handler._batch_render_chat = batch_render_chat
|
||||
handler._preprocess_cmpl_online = lambda *_args, **_kwargs: (
|
||||
pytest.fail("completion path should be skipped when a template exists")
|
||||
handler._preprocess_cmpl_online = lambda *_args, **_kwargs: pytest.fail(
|
||||
"completion path should be skipped when a template exists"
|
||||
)
|
||||
|
||||
handler._pre_process_cohere_online(ctx)
|
||||
|
||||
@@ -17,10 +17,10 @@ from vllm.entrypoints.serve.disagg.protocol import (
|
||||
GenerateResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
@@ -92,10 +92,9 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -103,7 +102,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
|
||||
serving = ServingTokens(
|
||||
engine,
|
||||
models,
|
||||
openai_serving_render=serving_render,
|
||||
online_renderer=online_renderer,
|
||||
request_logger=None,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -111,7 +110,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
|
||||
async def _fake_preprocess(*args, **kwargs):
|
||||
return [{"prompt_token_ids": [1, 2, 3]}]
|
||||
|
||||
serving.openai_serving_render.preprocess_completion = AsyncMock(
|
||||
serving.online_renderer.preprocess_completion = AsyncMock(
|
||||
side_effect=_fake_preprocess
|
||||
)
|
||||
return serving
|
||||
@@ -199,9 +198,7 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert (
|
||||
serving.openai_serving_render.preprocess_completion.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@@ -10,12 +10,12 @@ import pytest
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
TokenizeChatRequest,
|
||||
TokenizeCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
@@ -63,18 +63,16 @@ def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization:
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
return ServingTokenization(
|
||||
models,
|
||||
openai_serving_render=serving_render,
|
||||
request_logger=None,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
@@ -89,7 +87,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
|
||||
mock_engine.renderer = MagicMock()
|
||||
|
||||
serving = _build_serving_tokenization(mock_engine)
|
||||
serving.openai_serving_render.preprocess_chat = AsyncMock(
|
||||
serving.online_renderer.preprocess_chat = AsyncMock(
|
||||
return_value=(
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
@@ -105,7 +103,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
|
||||
|
||||
assert response.tokens == [1, 2, 3]
|
||||
assert (
|
||||
serving.openai_serving_render.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
@@ -119,7 +117,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
|
||||
mock_engine.renderer = MagicMock()
|
||||
|
||||
serving = _build_serving_tokenization(mock_engine)
|
||||
serving.openai_serving_render.preprocess_completion = AsyncMock(
|
||||
serving.online_renderer.preprocess_completion = AsyncMock(
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
@@ -132,8 +130,6 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
|
||||
|
||||
assert response.tokens == [1, 2, 3]
|
||||
assert (
|
||||
serving.openai_serving_render.preprocess_completion.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
@@ -99,8 +99,8 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
|
||||
engine_client = SimpleNamespace(
|
||||
errored=False,
|
||||
generate=Mock(
|
||||
side_effect=lambda *_args, **_kwargs: (
|
||||
_records_start_then_never_finishes(started_request_ids, _args[2])
|
||||
side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes(
|
||||
started_request_ids, _args[2]
|
||||
)
|
||||
),
|
||||
abort=AsyncMock(),
|
||||
|
||||
Reference in New Issue
Block a user