[Frontend] Split ServingRender into renderer and entrypoint. (#44285)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-06-23 11:19:09 +00:00
committed by GitHub
parent d32575a2d2
commit 31ca9504b1
26 changed files with 1155 additions and 939 deletions
@@ -17,9 +17,11 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
@@ -88,19 +90,19 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_chat = OpenAIServingChat(
engine,
models,
response_role="assistant",
openai_serving_render=serving_render,
online_renderer=online_renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -113,7 +115,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
[{"prompt_token_ids": [1, 2, 3]}],
)
serving_chat.openai_serving_render.preprocess_chat = AsyncMock(
serving_chat.online_renderer.preprocess_chat = AsyncMock(
side_effect=_fake_preprocess_chat
)
return serving_chat
@@ -187,13 +189,46 @@ async def test_openai_chat_keeps_mm_cache_for_engine_execution():
assert isinstance(result, tuple)
assert (
serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
"skip_mm_cache"
]
serving_chat.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is False
)
def _build_serving_render(engine: AsyncLLM) -> ServingRender:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
online_derenderer = OnlineDerenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_render = ServingRender(models, online_renderer, online_derenderer)
async def _fake_preprocess_chat(*args, **kwargs):
# return conversation, engine_inputs
return (
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
)
serving_render.online_renderer.preprocess_chat = AsyncMock(
side_effect=_fake_preprocess_chat
)
return serving_render
@pytest.mark.asyncio
async def test_renderer_only_chat_request_skips_mm_cache():
mock_engine = MagicMock(spec=AsyncLLM)
@@ -202,20 +237,18 @@ async def test_renderer_only_chat_request_skips_mm_cache():
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_chat = _build_serving_chat(mock_engine)
serving_render = _build_serving_render(mock_engine)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
)
result = await serving_chat.openai_serving_render.render_chat_request(request)
result = await serving_render.render_chat_request(request)
assert result.token_ids == [1, 2, 3]
assert (
serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
"skip_mm_cache"
]
serving_render.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is True
)
@@ -38,7 +38,6 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIServingModels,
)
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.exceptions import VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.multimodal.inputs import PlaceholderRange
@@ -46,6 +45,7 @@ from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser import HarmonyParser
from vllm.renderers.hf import HfRenderer
from vllm.renderers.mistral import MistralRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.registry import cached_tokenizer_from_config
@@ -567,13 +567,12 @@ def _build_renderer(model_config: MockModelConfig):
)
def _build_serving_render(
def _build_online_renderer(
engine, model_registry: OpenAIModelRegistry
) -> OpenAIServingRender:
return OpenAIServingRender(
) -> OnlineRenderer:
return OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=model_registry,
request_logger=None,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
@@ -591,13 +590,13 @@ def _build_serving_chat(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
openai_serving_render = _build_serving_render(engine, models.registry)
online_renderer = _build_online_renderer(engine, models.registry)
serving_chat = OpenAIServingChat(
engine,
models,
response_role="assistant",
openai_serving_render=openai_serving_render,
online_renderer=online_renderer,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
@@ -621,13 +620,13 @@ async def _async_serving_chat_init():
engine = MockEngine()
models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
openai_serving_render = _build_serving_render(engine, models.registry)
online_renderer = _build_online_renderer(engine, models.registry)
serving_completion = OpenAIServingChat(
engine,
models,
response_role="assistant",
openai_serving_render=openai_serving_render,
online_renderer=online_renderer,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
@@ -1396,9 +1395,7 @@ class TestServingChatWithHarmony:
messages=messages,
include_reasoning=include_reasoning,
)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
[
@@ -1429,8 +1426,8 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the second turn's input
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
input_messages_2, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
req_2
)
verify_harmony_messages(
input_messages_2,
@@ -1452,9 +1449,7 @@ class TestServingChatWithHarmony:
{"role": "user", "content": "Hello"},
]
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
[
@@ -1478,9 +1473,7 @@ class TestServingChatWithHarmony:
req = ChatCompletionRequest(
model=MODEL_NAME, messages=messages, tools=weather_tools
)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
[
@@ -1503,9 +1496,7 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the first turn's input
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
[
@@ -1549,8 +1540,8 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the second turn's input
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages_2, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
req_2
)
verify_harmony_messages(
input_messages_2,
@@ -1588,9 +1579,7 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the first turn's input
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
[
@@ -1634,8 +1623,8 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the second turn's input
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages_2, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
req_2
)
verify_harmony_messages(
input_messages_2,
@@ -1686,8 +1675,8 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the third turn's input
req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages_3, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req_3)
input_messages_3, _ = serving_chat.online_renderer._make_request_with_harmony(
req_3
)
verify_harmony_messages(
input_messages_3,
@@ -1751,8 +1740,8 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the fourth turn's input
req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages_4, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req_4)
input_messages_4, _ = serving_chat.online_renderer._make_request_with_harmony(
req_4
)
verify_harmony_messages(
input_messages_4,
@@ -1802,9 +1791,7 @@ class TestServingChatWithHarmony:
},
]
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
@@ -1835,9 +1822,7 @@ class TestServingChatWithHarmony:
},
]
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
@@ -1866,9 +1851,7 @@ class TestServingChatWithHarmony:
},
]
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
@@ -1898,14 +1881,14 @@ async def test_tool_choice_validation_without_parser():
engine_client=mock_engine,
base_model_paths=BASE_MODEL_PATHS,
)
openai_serving_render = _build_serving_render(mock_engine, models.registry)
online_renderer = _build_online_renderer(mock_engine, models.registry)
# Create serving_chat without tool_parser (enable_auto_tools=False)
serving_chat = OpenAIServingChat(
mock_engine,
models,
response_role="assistant",
openai_serving_render=openai_serving_render,
online_renderer=online_renderer,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
@@ -1967,13 +1950,13 @@ async def test_streaming_n_gt1_independent_tool_parsers():
engine_client=mock_engine,
base_model_paths=BASE_MODEL_PATHS,
)
openai_serving_render = _build_serving_render(mock_engine, models.registry)
online_renderer = _build_online_renderer(mock_engine, models.registry)
serving_chat = OpenAIServingChat(
mock_engine,
models,
response_role="assistant",
openai_serving_render=openai_serving_render,
online_renderer=online_renderer,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
@@ -14,9 +14,11 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
@@ -77,10 +79,9 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -88,7 +89,7 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
return OpenAIServingCompletion(
engine,
models,
openai_serving_render=serving_render,
online_renderer=online_renderer,
request_logger=None,
)
@@ -158,7 +159,7 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_completion = _build_serving_completion(mock_engine)
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
serving_completion.online_renderer.preprocess_completion = AsyncMock(
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
@@ -171,13 +172,48 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
assert isinstance(result, list)
assert (
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
serving_completion.online_renderer.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
is False
)
def _build_serving_render(engine: AsyncLLM) -> ServingRender:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
online_derenderer = OnlineDerenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_render = ServingRender(models, online_renderer, online_derenderer)
async def _fake_preprocess_chat(*args, **kwargs):
# return conversation, engine_inputs
return (
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
)
serving_render.online_renderer.preprocess_chat = AsyncMock(
side_effect=_fake_preprocess_chat
)
return serving_render
@pytest.mark.asyncio
async def test_renderer_only_completion_request_skips_mm_cache():
mock_engine = MagicMock(spec=AsyncLLM)
@@ -186,8 +222,9 @@ async def test_renderer_only_completion_request_skips_mm_cache():
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_completion = _build_serving_completion(mock_engine)
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
serving_render = _build_serving_render(mock_engine)
serving_render.online_renderer.preprocess_completion = AsyncMock(
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
@@ -196,13 +233,11 @@ async def test_renderer_only_completion_request_skips_mm_cache():
prompt="Test prompt",
)
result = await serving_completion.openai_serving_render.render_completion_request(
request
)
result = await serving_render.render_completion_request(request)
assert isinstance(result, list)
assert (
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
serving_render.online_renderer.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
is True
@@ -14,10 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.lora.request import LoRARequest
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
@@ -144,16 +144,15 @@ def mock_serving_setup():
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=mock_engine.model_config,
renderer=mock_engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_completion = OpenAIServingCompletion(
mock_engine, models, openai_serving_render=serving_render, request_logger=None
mock_engine, models, online_renderer=online_renderer, request_logger=None
)
return mock_engine, serving_completion
@@ -230,7 +230,7 @@ class TestInitializeToolSessions:
instance = OpenAIServingResponses(
engine_client=engine_client,
models=models,
openai_serving_render=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -316,7 +316,7 @@ class TestValidateGeneratorInput:
instance = OpenAIServingResponses(
engine_client=engine_client,
models=models,
openai_serving_render=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -379,7 +379,7 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
serving = OpenAIServingResponses(
engine_client=engine_client,
models=models,
openai_serving_render=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -684,7 +684,7 @@ def _make_serving_instance_with_reasoning():
serving = OpenAIServingResponses(
engine_client=engine_client,
models=models,
openai_serving_render=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -428,8 +428,8 @@ class TestPreProcessCohereOnline:
handler._get_task_instruction_prefix = lambda _input_type: None
handler._has_chat_template = lambda: False
handler._preprocess_cmpl_online = preprocess_cmpl_online
handler._batch_render_chat = lambda *_args, **_kwargs: (
pytest.fail("text-only request should not require chat rendering")
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
"text-only request should not require chat rendering"
)
handler._pre_process_cohere_online(ctx)
@@ -448,8 +448,8 @@ class TestPreProcessCohereOnline:
handler._get_task_instruction_prefix = lambda _input_type: "query: "
handler._has_chat_template = lambda: False
handler._batch_render_chat = lambda *_args, **_kwargs: (
pytest.fail("chat rendering should be skipped without a template")
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
"chat rendering should be skipped without a template"
)
handler._preprocess_cmpl_online = preprocess_cmpl
@@ -485,8 +485,8 @@ class TestPreProcessCohereOnline:
handler._get_task_instruction_prefix = lambda _input_type: "query: "
handler._has_chat_template = lambda: True
handler._batch_render_chat = batch_render_chat
handler._preprocess_cmpl_online = lambda *_args, **_kwargs: (
pytest.fail("completion path should be skipped when a template exists")
handler._preprocess_cmpl_online = lambda *_args, **_kwargs: pytest.fail(
"completion path should be skipped when a template exists"
)
handler._pre_process_cohere_online(ctx)
@@ -17,10 +17,10 @@ from vllm.entrypoints.serve.disagg.protocol import (
GenerateResponse,
)
from vllm.entrypoints.serve.disagg.serving import ServingTokens
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.async_llm import AsyncLLM
@@ -92,10 +92,9 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -103,7 +102,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
serving = ServingTokens(
engine,
models,
openai_serving_render=serving_render,
online_renderer=online_renderer,
request_logger=None,
**kwargs,
)
@@ -111,7 +110,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
async def _fake_preprocess(*args, **kwargs):
return [{"prompt_token_ids": [1, 2, 3]}]
serving.openai_serving_render.preprocess_completion = AsyncMock(
serving.online_renderer.preprocess_completion = AsyncMock(
side_effect=_fake_preprocess
)
return serving
@@ -199,9 +198,7 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
assert isinstance(response, GenerateResponse)
assert (
serving.openai_serving_render.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
is True
)
@@ -10,12 +10,12 @@ import pytest
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.tokenize.protocol import (
TokenizeChatRequest,
TokenizeCompletionRequest,
)
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.v1.engine.async_llm import AsyncLLM
MODEL_NAME = "openai-community/gpt2"
@@ -63,18 +63,16 @@ def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization:
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
return ServingTokenization(
models,
openai_serving_render=serving_render,
request_logger=None,
online_renderer=online_renderer,
chat_template=None,
chat_template_content_format="auto",
)
@@ -89,7 +87,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
mock_engine.renderer = MagicMock()
serving = _build_serving_tokenization(mock_engine)
serving.openai_serving_render.preprocess_chat = AsyncMock(
serving.online_renderer.preprocess_chat = AsyncMock(
return_value=(
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
@@ -105,7 +103,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
assert response.tokens == [1, 2, 3]
assert (
serving.openai_serving_render.preprocess_chat.call_args.kwargs["skip_mm_cache"]
serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is True
)
@@ -119,7 +117,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
mock_engine.renderer = MagicMock()
serving = _build_serving_tokenization(mock_engine)
serving.openai_serving_render.preprocess_completion = AsyncMock(
serving.online_renderer.preprocess_completion = AsyncMock(
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
@@ -132,8 +130,6 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
assert response.tokens == [1, 2, 3]
assert (
serving.openai_serving_render.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
is True
)
@@ -99,8 +99,8 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
engine_client = SimpleNamespace(
errored=False,
generate=Mock(
side_effect=lambda *_args, **_kwargs: (
_records_start_then_never_finishes(started_request_ids, _args[2])
side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes(
started_request_ids, _args[2]
)
),
abort=AsyncMock(),