[Frontend] Split ServingRender into renderer and entrypoint. (#44285)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-06-23 11:19:09 +00:00
committed by GitHub
parent d32575a2d2
commit 31ca9504b1
26 changed files with 1155 additions and 939 deletions
@@ -17,9 +17,11 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
@@ -88,19 +90,19 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_chat = OpenAIServingChat(
engine,
models,
response_role="assistant",
openai_serving_render=serving_render,
online_renderer=online_renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -113,7 +115,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
[{"prompt_token_ids": [1, 2, 3]}],
)
serving_chat.openai_serving_render.preprocess_chat = AsyncMock(
serving_chat.online_renderer.preprocess_chat = AsyncMock(
side_effect=_fake_preprocess_chat
)
return serving_chat
@@ -187,13 +189,46 @@ async def test_openai_chat_keeps_mm_cache_for_engine_execution():
assert isinstance(result, tuple)
assert (
serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
"skip_mm_cache"
]
serving_chat.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is False
)
def _build_serving_render(engine: AsyncLLM) -> ServingRender:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
online_derenderer = OnlineDerenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_render = ServingRender(models, online_renderer, online_derenderer)
async def _fake_preprocess_chat(*args, **kwargs):
# return conversation, engine_inputs
return (
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
)
serving_render.online_renderer.preprocess_chat = AsyncMock(
side_effect=_fake_preprocess_chat
)
return serving_render
@pytest.mark.asyncio
async def test_renderer_only_chat_request_skips_mm_cache():
mock_engine = MagicMock(spec=AsyncLLM)
@@ -202,20 +237,18 @@ async def test_renderer_only_chat_request_skips_mm_cache():
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_chat = _build_serving_chat(mock_engine)
serving_render = _build_serving_render(mock_engine)
request = ChatCompletionRequest(
model=MODEL_NAME,
messages=[{"role": "user", "content": "Test prompt"}],
)
result = await serving_chat.openai_serving_render.render_chat_request(request)
result = await serving_render.render_chat_request(request)
assert result.token_ids == [1, 2, 3]
assert (
serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
"skip_mm_cache"
]
serving_render.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is True
)
@@ -38,7 +38,6 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIServingModels,
)
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.exceptions import VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.multimodal.inputs import PlaceholderRange
@@ -46,6 +45,7 @@ from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser import HarmonyParser
from vllm.renderers.hf import HfRenderer
from vllm.renderers.mistral import MistralRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.registry import cached_tokenizer_from_config
@@ -567,13 +567,12 @@ def _build_renderer(model_config: MockModelConfig):
)
def _build_serving_render(
def _build_online_renderer(
engine, model_registry: OpenAIModelRegistry
) -> OpenAIServingRender:
return OpenAIServingRender(
) -> OnlineRenderer:
return OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=model_registry,
request_logger=None,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
@@ -591,13 +590,13 @@ def _build_serving_chat(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
openai_serving_render = _build_serving_render(engine, models.registry)
online_renderer = _build_online_renderer(engine, models.registry)
serving_chat = OpenAIServingChat(
engine,
models,
response_role="assistant",
openai_serving_render=openai_serving_render,
online_renderer=online_renderer,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
@@ -621,13 +620,13 @@ async def _async_serving_chat_init():
engine = MockEngine()
models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
openai_serving_render = _build_serving_render(engine, models.registry)
online_renderer = _build_online_renderer(engine, models.registry)
serving_completion = OpenAIServingChat(
engine,
models,
response_role="assistant",
openai_serving_render=openai_serving_render,
online_renderer=online_renderer,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
@@ -1396,9 +1395,7 @@ class TestServingChatWithHarmony:
messages=messages,
include_reasoning=include_reasoning,
)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
[
@@ -1429,8 +1426,8 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the second turn's input
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
input_messages_2, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
req_2
)
verify_harmony_messages(
input_messages_2,
@@ -1452,9 +1449,7 @@ class TestServingChatWithHarmony:
{"role": "user", "content": "Hello"},
]
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
[
@@ -1478,9 +1473,7 @@ class TestServingChatWithHarmony:
req = ChatCompletionRequest(
model=MODEL_NAME, messages=messages, tools=weather_tools
)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
[
@@ -1503,9 +1496,7 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the first turn's input
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
[
@@ -1549,8 +1540,8 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the second turn's input
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages_2, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
req_2
)
verify_harmony_messages(
input_messages_2,
@@ -1588,9 +1579,7 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the first turn's input
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
[
@@ -1634,8 +1623,8 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the second turn's input
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages_2, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
req_2
)
verify_harmony_messages(
input_messages_2,
@@ -1686,8 +1675,8 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the third turn's input
req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages_3, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req_3)
input_messages_3, _ = serving_chat.online_renderer._make_request_with_harmony(
req_3
)
verify_harmony_messages(
input_messages_3,
@@ -1751,8 +1740,8 @@ class TestServingChatWithHarmony:
# Test the Harmony messages for the fourth turn's input
req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
input_messages_4, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req_4)
input_messages_4, _ = serving_chat.online_renderer._make_request_with_harmony(
req_4
)
verify_harmony_messages(
input_messages_4,
@@ -1802,9 +1791,7 @@ class TestServingChatWithHarmony:
},
]
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
@@ -1835,9 +1822,7 @@ class TestServingChatWithHarmony:
},
]
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
@@ -1866,9 +1851,7 @@ class TestServingChatWithHarmony:
},
]
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
input_messages, _ = (
serving_chat.openai_serving_render._make_request_with_harmony(req)
)
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
verify_harmony_messages(
input_messages,
@@ -1898,14 +1881,14 @@ async def test_tool_choice_validation_without_parser():
engine_client=mock_engine,
base_model_paths=BASE_MODEL_PATHS,
)
openai_serving_render = _build_serving_render(mock_engine, models.registry)
online_renderer = _build_online_renderer(mock_engine, models.registry)
# Create serving_chat without tool_parser (enable_auto_tools=False)
serving_chat = OpenAIServingChat(
mock_engine,
models,
response_role="assistant",
openai_serving_render=openai_serving_render,
online_renderer=online_renderer,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
@@ -1967,13 +1950,13 @@ async def test_streaming_n_gt1_independent_tool_parsers():
engine_client=mock_engine,
base_model_paths=BASE_MODEL_PATHS,
)
openai_serving_render = _build_serving_render(mock_engine, models.registry)
online_renderer = _build_online_renderer(mock_engine, models.registry)
serving_chat = OpenAIServingChat(
mock_engine,
models,
response_role="assistant",
openai_serving_render=openai_serving_render,
online_renderer=online_renderer,
chat_template=CHAT_TEMPLATE,
chat_template_content_format="auto",
request_logger=None,
@@ -14,9 +14,11 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
@@ -77,10 +79,9 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -88,7 +89,7 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
return OpenAIServingCompletion(
engine,
models,
openai_serving_render=serving_render,
online_renderer=online_renderer,
request_logger=None,
)
@@ -158,7 +159,7 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_completion = _build_serving_completion(mock_engine)
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
serving_completion.online_renderer.preprocess_completion = AsyncMock(
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
@@ -171,13 +172,48 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
assert isinstance(result, list)
assert (
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
serving_completion.online_renderer.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
is False
)
def _build_serving_render(engine: AsyncLLM) -> ServingRender:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
online_derenderer = OnlineDerenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_render = ServingRender(models, online_renderer, online_derenderer)
async def _fake_preprocess_chat(*args, **kwargs):
# return conversation, engine_inputs
return (
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
)
serving_render.online_renderer.preprocess_chat = AsyncMock(
side_effect=_fake_preprocess_chat
)
return serving_render
@pytest.mark.asyncio
async def test_renderer_only_completion_request_skips_mm_cache():
mock_engine = MagicMock(spec=AsyncLLM)
@@ -186,8 +222,9 @@ async def test_renderer_only_completion_request_skips_mm_cache():
mock_engine.input_processor = MagicMock()
mock_engine.renderer = _build_renderer(mock_engine.model_config)
serving_completion = _build_serving_completion(mock_engine)
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
serving_render = _build_serving_render(mock_engine)
serving_render.online_renderer.preprocess_completion = AsyncMock(
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
@@ -196,13 +233,11 @@ async def test_renderer_only_completion_request_skips_mm_cache():
prompt="Test prompt",
)
result = await serving_completion.openai_serving_render.render_completion_request(
request
)
result = await serving_render.render_completion_request(request)
assert isinstance(result, list)
assert (
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
serving_render.online_renderer.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
is True
@@ -14,10 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.lora.request import LoRARequest
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.engine.async_llm import AsyncLLM
@@ -144,16 +144,15 @@ def mock_serving_setup():
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=mock_engine.model_config,
renderer=mock_engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving_completion = OpenAIServingCompletion(
mock_engine, models, openai_serving_render=serving_render, request_logger=None
mock_engine, models, online_renderer=online_renderer, request_logger=None
)
return mock_engine, serving_completion
@@ -230,7 +230,7 @@ class TestInitializeToolSessions:
instance = OpenAIServingResponses(
engine_client=engine_client,
models=models,
openai_serving_render=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -316,7 +316,7 @@ class TestValidateGeneratorInput:
instance = OpenAIServingResponses(
engine_client=engine_client,
models=models,
openai_serving_render=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -379,7 +379,7 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
serving = OpenAIServingResponses(
engine_client=engine_client,
models=models,
openai_serving_render=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -684,7 +684,7 @@ def _make_serving_instance_with_reasoning():
serving = OpenAIServingResponses(
engine_client=engine_client,
models=models,
openai_serving_render=MagicMock(),
online_renderer=MagicMock(),
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -428,8 +428,8 @@ class TestPreProcessCohereOnline:
handler._get_task_instruction_prefix = lambda _input_type: None
handler._has_chat_template = lambda: False
handler._preprocess_cmpl_online = preprocess_cmpl_online
handler._batch_render_chat = lambda *_args, **_kwargs: (
pytest.fail("text-only request should not require chat rendering")
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
"text-only request should not require chat rendering"
)
handler._pre_process_cohere_online(ctx)
@@ -448,8 +448,8 @@ class TestPreProcessCohereOnline:
handler._get_task_instruction_prefix = lambda _input_type: "query: "
handler._has_chat_template = lambda: False
handler._batch_render_chat = lambda *_args, **_kwargs: (
pytest.fail("chat rendering should be skipped without a template")
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
"chat rendering should be skipped without a template"
)
handler._preprocess_cmpl_online = preprocess_cmpl
@@ -485,8 +485,8 @@ class TestPreProcessCohereOnline:
handler._get_task_instruction_prefix = lambda _input_type: "query: "
handler._has_chat_template = lambda: True
handler._batch_render_chat = batch_render_chat
handler._preprocess_cmpl_online = lambda *_args, **_kwargs: (
pytest.fail("completion path should be skipped when a template exists")
handler._preprocess_cmpl_online = lambda *_args, **_kwargs: pytest.fail(
"completion path should be skipped when a template exists"
)
handler._pre_process_cohere_online(ctx)
@@ -17,10 +17,10 @@ from vllm.entrypoints.serve.disagg.protocol import (
GenerateResponse,
)
from vllm.entrypoints.serve.disagg.serving import ServingTokens
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.async_llm import AsyncLLM
@@ -92,10 +92,9 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -103,7 +102,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
serving = ServingTokens(
engine,
models,
openai_serving_render=serving_render,
online_renderer=online_renderer,
request_logger=None,
**kwargs,
)
@@ -111,7 +110,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
async def _fake_preprocess(*args, **kwargs):
return [{"prompt_token_ids": [1, 2, 3]}]
serving.openai_serving_render.preprocess_completion = AsyncMock(
serving.online_renderer.preprocess_completion = AsyncMock(
side_effect=_fake_preprocess
)
return serving
@@ -199,9 +198,7 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
assert isinstance(response, GenerateResponse)
assert (
serving.openai_serving_render.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
is True
)
@@ -10,12 +10,12 @@ import pytest
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.tokenize.protocol import (
TokenizeChatRequest,
TokenizeCompletionRequest,
)
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.v1.engine.async_llm import AsyncLLM
MODEL_NAME = "openai-community/gpt2"
@@ -63,18 +63,16 @@ def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization:
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
return ServingTokenization(
models,
openai_serving_render=serving_render,
request_logger=None,
online_renderer=online_renderer,
chat_template=None,
chat_template_content_format="auto",
)
@@ -89,7 +87,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
mock_engine.renderer = MagicMock()
serving = _build_serving_tokenization(mock_engine)
serving.openai_serving_render.preprocess_chat = AsyncMock(
serving.online_renderer.preprocess_chat = AsyncMock(
return_value=(
[{"role": "user", "content": "Test"}],
[{"prompt_token_ids": [1, 2, 3]}],
@@ -105,7 +103,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
assert response.tokens == [1, 2, 3]
assert (
serving.openai_serving_render.preprocess_chat.call_args.kwargs["skip_mm_cache"]
serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
is True
)
@@ -119,7 +117,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
mock_engine.renderer = MagicMock()
serving = _build_serving_tokenization(mock_engine)
serving.openai_serving_render.preprocess_completion = AsyncMock(
serving.online_renderer.preprocess_completion = AsyncMock(
return_value=[{"prompt_token_ids": [1, 2, 3]}]
)
@@ -132,8 +130,6 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
assert response.tokens == [1, 2, 3]
assert (
serving.openai_serving_render.preprocess_completion.call_args.kwargs[
"skip_mm_cache"
]
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
is True
)
@@ -99,8 +99,8 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
engine_client = SimpleNamespace(
errored=False,
generate=Mock(
side_effect=lambda *_args, **_kwargs: (
_records_start_then_never_finishes(started_request_ids, _args[2])
side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes(
started_request_ids, _args[2]
)
),
abort=AsyncMock(),
+3 -4
View File
@@ -512,12 +512,11 @@ async def test_header_dp_rank_argument():
)
# Create render serving instance (required by OpenAIServingChat)
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.renderers.online_renderer import OnlineRenderer
serving_render = OpenAIServingRender(
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
model_registry=models.registry,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
@@ -528,7 +527,7 @@ async def test_header_dp_rank_argument():
engine_client=engine,
models=models,
response_role="assistant",
openai_serving_render=serving_render,
online_renderer=online_renderer,
chat_template=None,
chat_template_content_format="auto",
request_logger=None,
+4 -6
View File
@@ -10,7 +10,7 @@ import logging
import time
import uuid
from collections.abc import AsyncGenerator
from typing import TYPE_CHECKING, Any
from typing import Any
import jinja2
from fastapi import Request
@@ -48,9 +48,7 @@ from vllm.entrypoints.openai.engine.protocol import (
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
if TYPE_CHECKING:
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.renderers.online_renderer import OnlineRenderer
logger = logging.getLogger(__name__)
@@ -111,7 +109,7 @@ class AnthropicServingMessages(OpenAIServingChat):
models: OpenAIServingModels,
response_role: str,
*,
openai_serving_render: "OpenAIServingRender",
online_renderer: "OnlineRenderer",
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
@@ -127,7 +125,7 @@ class AnthropicServingMessages(OpenAIServingChat):
engine_client=engine_client,
models=models,
response_role=response_role,
openai_serving_render=openai_serving_render,
online_renderer=online_renderer,
request_logger=request_logger,
chat_template=chat_template,
chat_template_content_format=chat_template_content_format,
+6 -6
View File
@@ -87,7 +87,7 @@ async def init_generate_state(
tool_server = None
resolved_chat_template = load_chat_template(args.chat_template)
# Render endpoints are always backed by OpenAIServingRender so that
# Render endpoints are always backed by OnlineRenderer so that
# /v1/chat/completions/render and /v1/completions/render work on both
# generate-mode and render-only servers. Created in init_app_state.
@@ -95,7 +95,7 @@ async def init_generate_state(
OpenAIServingResponses(
engine_client,
state.openai_serving_models,
state.openai_serving_render,
state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
@@ -116,7 +116,7 @@ async def init_generate_state(
engine_client=engine_client,
models=state.openai_serving_models,
response_role=args.response_role,
openai_serving_render=state.openai_serving_render,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
@@ -146,7 +146,7 @@ async def init_generate_state(
OpenAIServingCompletion(
engine_client,
state.openai_serving_models,
openai_serving_render=state.openai_serving_render,
online_renderer=state.online_renderer,
request_logger=request_logger,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
@@ -160,7 +160,7 @@ async def init_generate_state(
engine_client,
state.openai_serving_models,
args.response_role,
openai_serving_render=state.openai_serving_render,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
@@ -179,7 +179,7 @@ async def init_generate_state(
ServingTokens(
engine_client,
state.openai_serving_models,
state.openai_serving_render,
state.online_renderer,
request_logger=request_logger,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
+50 -8
View File
@@ -32,7 +32,7 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import (
@@ -55,6 +55,8 @@ from vllm.entrypoints.serve.utils.server_utils import (
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParserManager
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tasks import POOLING_TASKS, SupportedTask
from vllm.tool_parsers import ToolParserManager
from vllm.tracing import instrument
@@ -360,10 +362,24 @@ async def init_app_state(
)
await state.openai_serving_models.init_static_loras()
state.openai_serving_render = OpenAIServingRender(
state.online_renderer = OnlineRenderer(
model_config=engine_client.model_config,
renderer=engine_client.renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.online_derenderer = OnlineDerenderer(
model_config=engine_client.model_config,
renderer=engine_client.renderer,
model_registry=state.openai_serving_models.registry,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
@@ -378,13 +394,19 @@ async def init_app_state(
state.serving_tokenization = ServingTokenization(
state.openai_serving_models,
state.openai_serving_render,
state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
state.serving_render = ServingRender(
state.openai_serving_models,
state.online_renderer,
state.online_derenderer,
request_logger=request_logger,
)
if "generate" in supported_tasks:
from vllm.entrypoints.generate.api_router import init_generate_state
@@ -423,8 +445,8 @@ async def init_render_app_state(
"""
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
served_model_names = args.served_model_name or [args.model]
model_registry = OpenAIModelRegistry(
@@ -443,10 +465,24 @@ async def init_render_app_state(
renderer = renderer_from_config(vllm_config)
resolved_chat_template = load_chat_template(args.chat_template)
state.openai_serving_render = OpenAIServingRender(
state.online_renderer = OnlineRenderer(
model_config=vllm_config.model_config,
renderer=renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.online_derenderer = OnlineDerenderer(
model_config=vllm_config.model_config,
renderer=renderer,
model_registry=model_registry,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
@@ -462,13 +498,19 @@ async def init_render_app_state(
state.openai_serving_models = model_registry
state.serving_tokenization = ServingTokenization(
model_registry,
state.openai_serving_render,
state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
state.serving_render = ServingRender(
model_registry,
state.online_renderer,
state.online_derenderer,
request_logger=request_logger,
)
state.vllm_config = vllm_config
# Disable stats logging — there is no engine to poll.
@@ -47,7 +47,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
"""Validate the model and preprocess a batched chat completion request.
Performs engine-aware checks then delegates per-conversation
preprocessing to OpenAIServingRender, validating the chat template
preprocessing to OnlineRenderer, validating the chat template
once for the whole batch.
Returns:
@@ -62,19 +62,19 @@ class OpenAIServingChatBatch(OpenAIServingChat):
if self.engine_client.errored:
raise self.engine_client.dead_error
render = self.openai_serving_render
renderer = self.online_renderer
if not render.use_harmony:
if not renderer.use_harmony:
# Common case: validate the chat template once for the whole batch.
error_check_ret = render.validate_chat_template(
error_check_ret = renderer.validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=render.trust_request_chat_template,
trust_request_chat_template=renderer.trust_request_chat_template,
)
if error_check_ret is not None:
return error_check_ret
parser = render.parser
parser = renderer.parser
tool_dicts: list[dict] | None = None
all_conversations: list[list[ConversationMessage]] = []
@@ -82,17 +82,17 @@ class OpenAIServingChatBatch(OpenAIServingChat):
for messages in request.messages:
single_request = request.to_chat_completion_request(messages)
if render.use_harmony:
conversation, engine_prompts = render._make_request_with_harmony(
if renderer.use_harmony:
conversation, engine_prompts = renderer._make_request_with_harmony(
single_request, should_include_tools=tool_dicts is not None
)
else:
conversation, engine_prompts = await render.preprocess_chat(
conversation, engine_prompts = await renderer.preprocess_chat(
single_request,
messages,
default_template=render.chat_template,
default_template_content_format=render.chat_template_content_format,
default_template_kwargs=render.default_chat_template_kwargs,
default_template=renderer.chat_template,
default_template_content_format=renderer.chat_template_content_format,
default_template_kwargs=renderer.default_chat_template_kwargs,
tool_dicts=tool_dicts,
parser=parser,
)
@@ -7,7 +7,7 @@ import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, Final, cast
from typing import Any, Final, cast
import numpy as np
import pybase64 as base64
@@ -61,14 +61,12 @@ from vllm.outputs import RequestOutput
from vllm.parser import ParserManager
from vllm.parser.abstract_parser import Parser
from vllm.renderers import ChatParams
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.collection_utils import as_list
from vllm.utils.mistral import is_mistral_tool_parser
if TYPE_CHECKING:
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
logger = init_logger(__name__)
@@ -112,7 +110,7 @@ class OpenAIServingChat(OpenAIServing):
models: OpenAIServingModels,
response_role: str,
*,
openai_serving_render: "OpenAIServingRender",
online_renderer: "OnlineRenderer",
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
@@ -135,7 +133,7 @@ class OpenAIServingChat(OpenAIServing):
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.openai_serving_render = openai_serving_render
self.online_renderer = online_renderer
self.response_role = response_role
self.chat_template = chat_template
self.chat_template_content_format: Final = chat_template_content_format
@@ -212,7 +210,7 @@ class OpenAIServingChat(OpenAIServing):
"""
Validate the model and preprocess a chat completion request.
Delegates preprocessing logic to OpenAIServingRender, adding the
Delegates preprocessing logic to OnlineRenderer, adding the
engine-aware checks (LoRA model validation, engine health).
Returns:
@@ -230,7 +228,7 @@ class OpenAIServingChat(OpenAIServing):
if self.engine_client.errored:
raise self.engine_client.dead_error
return await self.openai_serving_render.render_chat(request)
return await self.online_renderer.render_chat(request)
async def create_chat_completion(
self,
@@ -6,7 +6,7 @@ import io
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import TYPE_CHECKING, cast
from typing import cast
import numpy as np
import pybase64 as base64
@@ -41,14 +41,12 @@ from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.outputs import RequestOutput
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.async_utils import merge_async_iterators
from vllm.utils.collection_utils import as_list
if TYPE_CHECKING:
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
logger = init_logger(__name__)
@@ -58,7 +56,7 @@ class OpenAIServingCompletion(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
openai_serving_render: "OpenAIServingRender",
online_renderer: "OnlineRenderer",
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
enable_prompt_tokens_details: bool = False,
@@ -71,7 +69,7 @@ class OpenAIServingCompletion(OpenAIServing):
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.openai_serving_render = openai_serving_render
self.online_renderer = online_renderer
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_force_include_usage = enable_force_include_usage
@@ -90,7 +88,7 @@ class OpenAIServingCompletion(OpenAIServing):
"""
Validate the model and preprocess a completion request.
Delegates preprocessing logic to OpenAIServingRender, adding the
Delegates preprocessing logic to OnlineRenderer, adding the
engine-aware checks (LoRA model validation, engine health).
Returns:
@@ -106,7 +104,7 @@ class OpenAIServingCompletion(OpenAIServing):
if self.engine_client.errored:
raise self.engine_client.dead_error
return await self.openai_serving_render.render_completion(request)
return await self.online_renderer.render_completion(request)
async def create_completion(
self,
+2 -6
View File
@@ -13,12 +13,8 @@ from starlette.datastructures import Headers
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
GenerationError,
+5 -5
View File
@@ -92,7 +92,6 @@ from vllm.entrypoints.openai.responses.utils import (
extract_function_tool_names,
extract_tool_types,
)
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import VLLMValidationError
@@ -103,6 +102,7 @@ from vllm.logprobs import SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.outputs import CompletionOutput
from vllm.parser import Parser, ParserManager
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.utils import random_uuid
@@ -155,7 +155,7 @@ class OpenAIServingResponses(OpenAIServing):
self,
engine_client: EngineClient,
models: OpenAIServingModels,
openai_serving_render: OpenAIServingRender,
online_renderer: OnlineRenderer,
*,
request_logger: RequestLogger | None,
chat_template: str | None,
@@ -177,7 +177,7 @@ class OpenAIServingResponses(OpenAIServing):
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.openai_serving_render = openai_serving_render
self.online_renderer = online_renderer
self.chat_template = chat_template
self.chat_template_content_format: Final = chat_template_content_format
self.chat_template_kwargs = default_chat_template_kwargs or {}
@@ -633,7 +633,7 @@ class OpenAIServingResponses(OpenAIServing):
prev_response_output=prev_response.output if prev_response else None,
)
chat_template_kwargs = self._effective_chat_template_kwargs(request)
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
_, engine_inputs = await self.online_renderer.preprocess_chat(
request,
messages,
default_template=self.chat_template,
@@ -657,7 +657,7 @@ class OpenAIServingResponses(OpenAIServing):
request_input=messages,
)
chat_template_kwargs = self._effective_chat_template_kwargs(request)
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
_, engine_inputs = await self.online_renderer.preprocess_chat(
request,
new_messages,
default_template=chat_template,
+1 -3
View File
@@ -10,9 +10,7 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Respons
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
)
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.disagg.protocol import (
GenerateRequest,
GenerateResponse,
+4 -4
View File
@@ -36,7 +36,6 @@ from vllm.entrypoints.serve.disagg.protocol import (
GenerateResponseStreamChoice,
GenerateStreamResponse,
)
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput, mm_input
@@ -48,6 +47,7 @@ from vllm.multimodal.inputs import (
PlaceholderRange,
)
from vllm.outputs import RequestOutput
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.utils.collection_utils import as_list
@@ -61,7 +61,7 @@ class ServingTokens(OpenAIServing):
self,
engine_client: EngineClient,
models: OpenAIServingModels,
openai_serving_render: OpenAIServingRender,
online_renderer: OnlineRenderer,
*,
request_logger: RequestLogger | None,
force_no_detokenize: bool = False,
@@ -75,7 +75,7 @@ class ServingTokens(OpenAIServing):
request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.openai_serving_render = openai_serving_render
self.online_renderer = online_renderer
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_log_outputs = enable_log_outputs
self.force_no_detokenize = force_no_detokenize
@@ -168,7 +168,7 @@ class ServingTokens(OpenAIServing):
cache_salt=request.cache_salt,
)
else:
(engine_input,) = await self.openai_serving_render.preprocess_completion(
(engine_input,) = await self.online_renderer.preprocess_completion(
request,
prompt_input=request.token_ids,
prompt_embeds=None,
+14 -3
View File
@@ -15,7 +15,12 @@ from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
from vllm.entrypoints.serve.disagg.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
GenerateRequest,
GenerateResponse,
)
from vllm.entrypoints.serve.tokenize.protocol import (
DetokenizeRequest,
TokenizeChatRequest,
@@ -45,11 +50,17 @@ class RendererChatRequest(RendererRequest, Protocol):
CompletionLikeRequest: TypeAlias = (
CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest
CompletionRequest
| TokenizeCompletionRequest
| DetokenizeRequest
| DerenderCompletionRequest
)
ChatLikeRequest: TypeAlias = (
ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest
ChatCompletionRequest
| BatchChatCompletionRequest
| TokenizeChatRequest
| DerenderChatRequest
)
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
+3 -3
View File
@@ -19,7 +19,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
DerenderCompletionRequest,
GenerateRequest,
)
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
@@ -28,8 +28,8 @@ logger = init_logger(__name__)
router = APIRouter()
def render(request: Request) -> OpenAIServingRender | None:
return getattr(request.app.state, "openai_serving_render", None)
def render(request: Request) -> ServingRender | None:
return getattr(request.app.state, "serving_render", None)
@router.post(
+129 -747
View File
@@ -1,44 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Sequence
from http import HTTPStatus
from typing import Any, cast
from typing import cast
from openai_harmony import Message as OpenAIMessage
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import (
ChatTemplateContentFormatOption,
ConversationMessage,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProbs,
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionResponseChoice,
ChatMessage,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionLogProbs,
CompletionRequest,
CompletionResponse,
CompletionResponseChoice,
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
ToolCall,
UsageInfo,
)
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
from vllm.entrypoints.openai.parser.harmony_utils import (
build_harmony_preamble,
extract_instructions_from_messages,
parse_chat_inputs_to_harmony_messages,
render_for_completion,
from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
from vllm.entrypoints.serve.disagg.protocol import (
DerenderChatRequest,
@@ -47,207 +27,49 @@ from vllm.entrypoints.serve.disagg.protocol import (
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import (
EngineInput,
MultiModalHashes,
MultiModalInput,
MultiModalPlaceholders,
PromptType,
SingletonPrompt,
tokens_input,
)
from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs.preprocess import (
extract_prompt_components,
extract_prompt_len,
parse_model_prompt,
prompt_to_seq,
)
from vllm.tokenizers import TokenizerLike
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.utils import random_uuid
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
from vllm.utils.mistral import mt as _mt
logger = init_logger(__name__)
def _parse_token_id_placeholder(token: str) -> int | None:
"""Extract token ID from a 'token_id:N' placeholder string."""
if not token.startswith("token_id:"):
return None
try:
return int(token[len("token_id:") :])
except ValueError:
return None
def _correct_decoded_token(
token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike
) -> str:
"""Use preceding tokens as context to fix U+FFFD from byte-fallback.
Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py.
"""
max_ctx = min(len(context_token_ids), 4)
for num_ctx in range(1, max_ctx + 1):
context = context_token_ids[-num_ctx:]
full_decoded = tokenizer.decode(context + [token_id])
if full_decoded.endswith("�"):
continue
clean_end = len(context)
for j in range(len(context) - 1, -1, -1):
if tokenizer.decode([context[j]]).endswith("�"):
clean_end = j
else:
break
clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else ""
if full_decoded.startswith(clean_prefix):
return full_decoded[len(clean_prefix) :]
common_len = 0
for a, b in zip(clean_prefix, full_decoded):
if a != b:
break
common_len += 1
return full_decoded[common_len:]
return ""
def _resolve_logprobs(
logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
) -> ChatCompletionLogProbs:
"""Resolve token_id:N placeholders in a ChatCompletionLogProbs object."""
if logprobs.content is None:
return logprobs
context_token_ids: list[int] = []
resolved_content = []
for entry in logprobs.content:
token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
sampled_id = _parse_token_id_placeholder(entry.token)
if token_str.endswith("�") and sampled_id is not None:
token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer)
token_bytes = list(token_str.encode("utf-8"))
resolved_top = []
for top in entry.top_logprobs:
top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
top_id = _parse_token_id_placeholder(top.token)
if top_str.endswith("�") and top_id is not None:
top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer)
top_bytes = list(top_str.encode("utf-8"))
resolved_top.append(
top.model_copy(update={"token": top_str, "bytes": top_bytes})
)
resolved_content.append(
entry.model_copy(
update={
"token": token_str,
"bytes": token_bytes,
"top_logprobs": resolved_top,
}
)
)
if sampled_id is not None:
context_token_ids.append(sampled_id)
return ChatCompletionLogProbs(content=resolved_content)
def _convert_chat_logprobs_to_completion_logprobs(
logprobs: ChatCompletionLogProbs,
) -> CompletionLogProbs:
"""Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
(parallel flat lists) as required by the /v1/completions response schema."""
if logprobs.content is None:
return CompletionLogProbs()
tokens: list[str] = []
token_logprobs: list[float | None] = []
top_logprobs_list: list[dict[str, float] | None] = []
text_offset: list[int] = []
offset = 0
for entry in logprobs.content:
text_offset.append(offset)
tokens.append(entry.token)
token_logprobs.append(entry.logprob)
top_logprobs_list.append(
{t.token: t.logprob for t in entry.top_logprobs}
if entry.top_logprobs
else None
)
offset += len(entry.token)
return CompletionLogProbs(
text_offset=text_offset,
token_logprobs=token_logprobs,
tokens=tokens,
top_logprobs=top_logprobs_list,
)
class OpenAIServingRender:
class ServingRender(BaseServing):
def __init__(
self,
model_config: ModelConfig,
renderer: BaseRenderer,
model_registry: OpenAIModelRegistry,
models: OpenAIServingModels | OpenAIModelRegistry,
online_renderer: "OnlineRenderer",
online_derenderer: "OnlineDerenderer",
*,
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
enable_auto_tools: bool = False,
exclude_tools_when_tool_choice_none: bool = False,
tool_parser: str | None = None,
reasoning_parser: str | None = None,
default_chat_template_kwargs: dict[str, Any] | None = None,
log_error_stack: bool = False,
request_logger: RequestLogger | None = None,
) -> None:
self.model_config = model_config
self.renderer = renderer
self.model_registry = model_registry
self.request_logger = request_logger
self.chat_template = chat_template
self.chat_template_content_format: ChatTemplateContentFormatOption = (
chat_template_content_format
super().__init__(
models=models,
model_config=online_renderer.model_config,
request_logger=request_logger,
)
self.trust_request_chat_template = trust_request_chat_template
self.enable_auto_tools = enable_auto_tools
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
self.parser: type[Parser] | None = ParserManager.get_parser(
tool_parser_name=tool_parser,
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
model_name=model_config.model,
is_harmony=self.use_harmony,
)
self.default_chat_template_kwargs: dict[str, Any] = (
default_chat_template_kwargs or {}
)
self.log_error_stack = log_error_stack
self.supports_browsing = False
self.supports_code_interpreter = False
self.default_sampling_params = model_config.get_diff_sampling_param()
mc = model_config
self.online_renderer = online_renderer
self.online_derenderer = online_derenderer
self.default_sampling_params = (
online_renderer.model_config.get_diff_sampling_param()
)
mc = online_renderer.model_config
self.override_max_tokens = (
self.default_sampling_params.get("max_tokens")
if mc.generation_config not in ("auto", "vllm")
@@ -273,7 +95,7 @@ class OpenAIServingRender:
"Beam search is not supported by the render endpoint"
)
result = await self.render_chat(request, skip_mm_cache=True)
result = await self.online_renderer.render_chat(request, skip_mm_cache=True)
if isinstance(result, ErrorResponse):
return result
@@ -319,90 +141,6 @@ class OpenAIServingRender:
priority=request.priority,
)
async def render_chat(
self,
request: ChatCompletionRequest,
*,
skip_mm_cache: bool = False,
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
"""Core preprocessing logic for chat requests (no model/engine check).
Called directly by render_chat_request and delegated to by
OpenAIServingChat.render_chat_request after its engine-aware checks.
"""
tokenizer = self.renderer.tokenizer
tool_parser = self.parser.tool_parser_cls if self.parser is not None else None
if is_mistral_tokenizer(tokenizer):
# because of issues with pydantic we need to potentially
# re-serialize the tool_calls field of the request
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
_mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
_mt.validate_request_params(request)
# Check if tool parsing is unavailable (common condition)
tool_parsing_unavailable = (
tool_parser is None
and not is_mistral_tokenizer(tokenizer)
and not self.use_harmony
)
# Validate tool_choice when tool parsing is required but unavailable
if tool_parsing_unavailable and request.tool_choice not in (
None,
"none",
):
if request.tool_choice == "auto" and not self.enable_auto_tools:
# for hf tokenizers, "auto" tools requires
# --enable-auto-tool-choice and --tool-call-parser
return self.create_error_response(
'"auto" tool choice requires '
"--enable-auto-tool-choice and --tool-call-parser to be set"
)
elif request.tool_choice != "auto":
# "required" or named tool requires tool parser
return self.create_error_response(
f'tool_choice="{request.tool_choice}" requires '
"--tool-call-parser to be set"
)
if request.tools is None or (
request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
):
tool_dicts = None
else:
tool_dicts = [tool.model_dump() for tool in request.tools]
if not self.use_harmony:
# Common case.
error_check_ret = self.validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if error_check_ret is not None:
return error_check_ret
conversation, engine_inputs = await self.preprocess_chat(
request,
request.messages,
default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=self.default_chat_template_kwargs,
tool_dicts=tool_dicts,
parser=self.parser,
skip_mm_cache=skip_mm_cache,
)
else:
# For GPT-OSS.
should_include_tools = tool_dicts is not None
conversation, engine_inputs = self._make_request_with_harmony(
request, should_include_tools
)
return conversation, engine_inputs
async def render_completion_request(
self,
request: CompletionRequest,
@@ -415,7 +153,9 @@ class OpenAIServingRender:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
result = await self.render_completion(request, skip_mm_cache=True)
result = await self.online_renderer.render_completion(
request, skip_mm_cache=True
)
if isinstance(result, ErrorResponse):
return result
generate_requests: list[GenerateRequest] = []
@@ -459,37 +199,116 @@ class OpenAIServingRender:
return generate_requests
async def render_completion(
async def derender_chat_response(
self,
request: CompletionRequest,
*,
skip_mm_cache: bool = False,
) -> list[EngineInput] | ErrorResponse:
"""Core preprocessing logic for completion requests (no model/engine check).
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
Called directly by render_completion_request and delegated to by
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
Non-streaming only: expects the complete GenerateResponse with all
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
When ``request.chat_request`` is provided, the parser splits the
output into (reasoning, content, tool_calls). Otherwise falls
back to plain detokenization.
"""
# Return error for unsupported features.
if request.suffix is not None:
return self.create_error_response("suffix is not currently supported")
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if request.echo and request.prompt_embeds is not None:
return self.create_error_response("Echo is unsupported with prompt embeds.")
if request.prompt_logprobs is not None and request.prompt_embeds is not None:
return self.create_error_response(
"prompt_logprobs is not compatible with prompt embeds."
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response, request.chat_request
)
except ValueError as exc:
return self.create_error_response(str(exc))
engine_inputs = await self.preprocess_completion(
request,
prompt_input=request.prompt,
prompt_embeds=request.prompt_embeds,
skip_mm_cache=skip_mm_cache,
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
gen = request.generate_response
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
return engine_inputs
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
request.model,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Non-streaming only. Mirrors the multi-prompt completions case: one
GenerateResponse per prompt, parallel to the list[GenerateRequest]
from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
(
choices,
total_prompt_tokens,
total_completion_tokens,
) = await self.online_derenderer.derender_completion(
request.generate_responses, request.prompt_tokens
)
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
request.model,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
)
@staticmethod
def _extract_mm_features(
@@ -529,440 +348,3 @@ class OpenAIServingRender:
mm_placeholders=mm_placeholders,
kwargs_data=kwargs_data,
)
def _make_request_with_harmony(
self,
request: ChatCompletionRequest,
should_include_tools: bool = True,
):
"""Build Harmony (GPT-OSS) messages and engine prompt from a chat request."""
messages: list[OpenAIMessage] = []
# because of issues with pydantic we need to potentially
# re-serialize the tool_calls field of the request
# for more info: see comment in `maybe_serialize_tool_calls`
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
chat_messages = list(request.messages)
instructions, chat_messages = extract_instructions_from_messages(chat_messages)
# Add system message.
# NOTE: In Chat Completion API, browsing is enabled by default
# if the model supports it. TODO: Support browsing.
assert not self.supports_browsing
assert not self.supports_code_interpreter
if (reasoning_effort := request.reasoning_effort) == "none":
raise ValueError(f"Harmony does not support {reasoning_effort=}")
tools = request.tools if should_include_tools else None
messages.extend(
build_harmony_preamble(
instructions=instructions,
tools=tools, # type: ignore[arg-type]
reasoning_effort=reasoning_effort,
with_custom_tools=should_include_tools,
)
)
# Add remaining conversation messages.
messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages))
# Render prompt token ids.
prompt_token_ids = render_for_completion(messages)
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
return messages, [engine_input]
async def derender_chat_response(
self,
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
Non-streaming only: expects the complete GenerateResponse with all
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
When ``request.chat_request`` is provided, the parser splits the
output into (reasoning, content, tool_calls). Otherwise falls
back to plain detokenization.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
tokenizer = self.renderer.get_tokenizer()
gen = request.generate_response
chat_request = request.chat_request
choices: list[ChatCompletionResponseChoice] = []
try:
for choice in gen.choices:
if not choice.token_ids:
raise ValueError(
f"choice {choice.index} has empty or null token_ids"
)
resolved_logprobs = (
_resolve_logprobs(choice.logprobs, tokenizer)
if choice.logprobs is not None
else None
)
if self.parser is not None and chat_request is not None:
# Parser path: decode with special tokens preserved
# so the parser can see markers like </think>,
# <tool_call>, or Harmony channel tokens.
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=False
)
chat_template_kwargs: dict[str, Any] = {}
if not self.use_harmony:
chat_template_kwargs = (
chat_request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)
parser = self.parser(
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
)
reasoning, content, tool_calls = parser.parse(
decoded_text,
chat_request,
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=choice.token_ids,
)
if not getattr(chat_request, "include_reasoning", True):
reasoning = None
tc_items = (
[
ToolCall(
id=random_uuid(),
function=tc,
)
for tc in tool_calls
]
if tool_calls
else []
)
message = ChatMessage(
role="assistant",
reasoning=reasoning,
content=content,
tool_calls=tc_items,
)
else:
# No parser: plain detokenization.
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
)
message = ChatMessage(role="assistant", content=decoded_text)
choices.append(
ChatCompletionResponseChoice(
index=choice.index,
message=message,
logprobs=resolved_logprobs,
finish_reason=choice.finish_reason,
)
)
except ValueError as exc:
return self.create_error_response(str(exc))
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
request.model,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Non-streaming only. Mirrors the multi-prompt completions case: one
GenerateResponse per prompt, parallel to the list[GenerateRequest]
from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
n = len(request.generate_responses)
prompt_tokens_list: list[int] = (
request.prompt_tokens if request.prompt_tokens is not None else [0] * n
)
tokenizer = self.renderer.get_tokenizer()
choices: list[CompletionResponseChoice] = []
total_prompt_tokens = 0
total_completion_tokens = 0
index = 0
for gen, pt in zip(request.generate_responses, prompt_tokens_list):
for choice in gen.choices:
if not choice.token_ids:
return self.create_error_response(
f"choice {choice.index} in response {gen.request_id} "
"has empty or null token_ids"
)
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
)
completion_logprobs = None
if choice.logprobs is not None:
resolved = _resolve_logprobs(choice.logprobs, tokenizer)
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
resolved
)
choices.append(
CompletionResponseChoice(
index=index,
text=decoded_text,
finish_reason=choice.finish_reason,
logprobs=completion_logprobs,
)
)
total_completion_tokens += len(choice.token_ids)
index += 1
total_prompt_tokens += pt
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
request.model,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
)
def create_error_response(
self,
message: str | Exception,
err_type: str = "BadRequestError",
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
param: str | None = None,
) -> ErrorResponse:
return create_error_response(message, err_type, status_code, param)
async def _check_model(
self,
request: Any,
) -> ErrorResponse | None:
return await self.model_registry.check_model(request.model)
def validate_chat_template(
self,
request_chat_template: str | None,
chat_template_kwargs: dict[str, Any] | None,
trust_request_chat_template: bool,
) -> ErrorResponse | None:
"""Copied from OpenAIServing._validate_chat_template."""
if not trust_request_chat_template and (
request_chat_template is not None
or (
chat_template_kwargs
and chat_template_kwargs.get("chat_template") is not None
)
):
return self.create_error_response(
"Chat template is passed with request, but "
"--trust-request-chat-template is not set. "
"Refused request with untrusted chat template."
)
return None
async def preprocess_completion(
self,
request: Any,
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
prompt_embeds: bytes | list[bytes] | None,
*,
skip_mm_cache: bool = False,
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_completion."""
prompts = list[SingletonPrompt | bytes]()
if prompt_embeds is not None: # embeds take higher priority
prompts.extend(prompt_to_seq(prompt_embeds))
if prompt_input is not None:
prompts.extend(prompt_to_seq(prompt_input))
return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache)
async def preprocess_cmpl(
self,
request: Any,
prompts: Sequence[PromptType | bytes],
*,
skip_mm_cache: bool = False,
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_cmpl."""
renderer = self.renderer
model_config = self.model_config
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(model_config, prompt)
)
for prompt in prompts
]
tok_params = request.build_tok_params(model_config)
return await renderer.render_cmpl_async(
parsed_prompts,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
skip_mm_cache=skip_mm_cache,
)
async def preprocess_chat(
self,
request: Any,
messages: list[Any],
default_template: str | None,
default_template_content_format: ChatTemplateContentFormatOption,
default_template_kwargs: dict[str, Any] | None,
tool_dicts: list[dict[str, Any]] | None = None,
parser: type[Parser] | None = None,
*,
skip_mm_cache: bool = False,
) -> tuple[list[ConversationMessage], list[EngineInput]]:
"""Copied from OpenAIServing._preprocess_chat."""
renderer = self.renderer
mm_config = self.model_config.multimodal_config
default_template_kwargs = merge_kwargs(
default_template_kwargs,
dict(
tools=tool_dicts,
tokenize=(
is_mistral_tokenizer(renderer.tokenizer)
or self.model_config.enable_prompt_embeds
),
),
)
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
default_template, default_template_content_format
).with_defaults(
default_template_kwargs,
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
)
(conversation,), (engine_input,) = await renderer.render_chat_async(
[messages],
chat_params,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
skip_mm_cache=skip_mm_cache,
)
# tool parsing is done only if a tool_parser has been set and if
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
# is set, we want to prevent parsing a tool_call hallucinated by the LLM
#
# Exception: Mistral grammar-capable tokenizers always call
# adjust_request — even for tool_choice="none" — so that the grammar
# factory can prevent special-token leakage.
if parser is not None:
tokenizer = renderer.get_tokenizer()
tool_parser = parser.tool_parser_cls
tool_choice = getattr(request, "tool_choice", "none")
is_mistral_grammar_eligible = (
tool_parser is not None
and is_mistral_tool_parser(tool_parser)
and is_mistral_tokenizer(tokenizer)
and tokenizer.supports_grammar
)
should_adjust_request = (
parser.reasoning_parser_cls is not None
or tool_choice != "none"
or is_mistral_grammar_eligible
)
if should_adjust_request:
if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
msg = (
"Tool usage is only supported "
"for Chat Completions API or Responses API requests, "
f"but got {type(request).__name__}"
)
raise NotImplementedError(msg)
request = parser(
tokenizer,
request.tools,
model_config=self.model_config,
chat_template_kwargs=chat_params.chat_template_kwargs,
).adjust_request(
request=request,
)
return conversation, [engine_input]
+9 -9
View File
@@ -12,7 +12,6 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIServingModels,
)
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.tokenize.protocol import (
DetokenizeRequest,
DetokenizeResponse,
@@ -24,6 +23,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import TokensPrompt, tokens_input
from vllm.logger import init_logger
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers import TokenizerLike
logger = init_logger(__name__)
@@ -33,22 +33,22 @@ class ServingTokenization(BaseServing):
def __init__(
self,
models: OpenAIServingModels | OpenAIModelRegistry,
openai_serving_render: OpenAIServingRender,
online_renderer: OnlineRenderer,
*,
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
default_chat_template_kwargs: dict[str, Any] | None = None,
trust_request_chat_template: bool = False,
request_logger: RequestLogger | None = None,
) -> None:
super().__init__(
models=models,
model_config=openai_serving_render.model_config,
model_config=online_renderer.model_config,
request_logger=request_logger,
)
self.renderer = openai_serving_render.renderer
self.openai_serving_render = openai_serving_render
self.renderer = online_renderer.renderer
self.online_renderer = online_renderer
self.chat_template = chat_template
self.chat_template_content_format: Final = chat_template_content_format
self.default_chat_template_kwargs = default_chat_template_kwargs or {}
@@ -73,7 +73,7 @@ class ServingTokenization(BaseServing):
if request.tools is None
else [tool.model_dump() for tool in request.tools]
)
error_check_ret = self.openai_serving_render.validate_chat_template(
error_check_ret = self.online_renderer.validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
@@ -81,7 +81,7 @@ class ServingTokenization(BaseServing):
if error_check_ret is not None:
return error_check_ret
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
_, engine_inputs = await self.online_renderer.preprocess_chat(
request,
request.messages,
default_template=self.chat_template,
@@ -91,7 +91,7 @@ class ServingTokenization(BaseServing):
skip_mm_cache=True,
)
else:
engine_inputs = await self.openai_serving_render.preprocess_completion(
engine_inputs = await self.online_renderer.preprocess_completion(
request,
prompt_input=request.prompt,
prompt_embeds=None,
+334
View File
@@ -0,0 +1,334 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProbs,
ChatCompletionRequest,
ChatCompletionResponseChoice,
ChatMessage,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionLogProbs,
CompletionResponseChoice,
)
from vllm.entrypoints.openai.engine.protocol import ToolCall
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
from vllm.entrypoints.serve.disagg.protocol import GenerateResponse
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer
from vllm.tokenizers import TokenizerLike
from vllm.utils import random_uuid
logger = init_logger(__name__)
class OnlineDerenderer:
def __init__(
self,
model_config: ModelConfig,
renderer: BaseRenderer,
*,
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
enable_auto_tools: bool = False,
exclude_tools_when_tool_choice_none: bool = False,
tool_parser: str | None = None,
reasoning_parser: str | None = None,
default_chat_template_kwargs: dict[str, Any] | None = None,
log_error_stack: bool = False,
) -> None:
self.model_config = model_config
self.renderer = renderer
self.request_logger = request_logger
self.enable_auto_tools = enable_auto_tools
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
self.parser: type[Parser] | None = ParserManager.get_parser(
tool_parser_name=tool_parser,
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
model_name=model_config.model,
is_harmony=self.use_harmony,
)
self.chat_template = chat_template
self.chat_template_content_format: ChatTemplateContentFormatOption = (
chat_template_content_format
)
self.default_chat_template_kwargs: dict[str, Any] = (
default_chat_template_kwargs or {}
)
self.trust_request_chat_template = trust_request_chat_template
self.log_error_stack = log_error_stack
self.supports_browsing = False
self.supports_code_interpreter = False
async def derender_chat(
self,
generate_response: GenerateResponse,
chat_request: ChatCompletionRequest | None = None,
) -> list[ChatCompletionResponseChoice]:
tokenizer = self.renderer.get_tokenizer()
choices: list[ChatCompletionResponseChoice] = []
for choice in generate_response.choices:
if not choice.token_ids:
raise ValueError(f"choice {choice.index} has empty or null token_ids")
resolved_logprobs = (
_resolve_logprobs(choice.logprobs, tokenizer)
if choice.logprobs is not None
else None
)
if self.parser is not None and chat_request is not None:
# Parser path: decode with special tokens preserved
# so the parser can see markers like </think>,
# <tool_call>, or Harmony channel tokens.
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=False
)
chat_template_kwargs: dict[str, Any] = {}
if not self.use_harmony:
chat_template_kwargs = (
chat_request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)
parser = self.parser(
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
)
reasoning, content, tool_calls = parser.parse(
decoded_text,
chat_request,
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=choice.token_ids,
)
if not getattr(chat_request, "include_reasoning", True):
reasoning = None
tc_items = (
[
ToolCall(
id=random_uuid(),
function=tc,
)
for tc in tool_calls
]
if tool_calls
else []
)
message = ChatMessage(
role="assistant",
reasoning=reasoning,
content=content,
tool_calls=tc_items,
)
else:
# No parser: plain detokenization.
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
)
message = ChatMessage(role="assistant", content=decoded_text)
choices.append(
ChatCompletionResponseChoice(
index=choice.index,
message=message,
logprobs=resolved_logprobs,
finish_reason=choice.finish_reason,
)
)
return choices
async def derender_completion(
self,
generate_responses: list[GenerateResponse],
prompt_tokens: list[int] | None = None,
) -> tuple[list[CompletionResponseChoice], int, int]:
n = len(generate_responses)
prompt_tokens_list: list[int] = (
prompt_tokens if prompt_tokens is not None else [0] * n
)
tokenizer = self.renderer.get_tokenizer()
choices: list[CompletionResponseChoice] = []
total_prompt_tokens = 0
total_completion_tokens = 0
index = 0
for gen, pt in zip(generate_responses, prompt_tokens_list):
for choice in gen.choices:
if not choice.token_ids:
raise ValueError(
f"choice {choice.index} in response {gen.request_id} "
"has empty or null token_ids"
)
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
)
completion_logprobs = None
if choice.logprobs is not None:
resolved = _resolve_logprobs(choice.logprobs, tokenizer)
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
resolved
)
choices.append(
CompletionResponseChoice(
index=index,
text=decoded_text,
finish_reason=choice.finish_reason,
logprobs=completion_logprobs,
)
)
total_completion_tokens += len(choice.token_ids)
index += 1
total_prompt_tokens += pt
return choices, total_prompt_tokens, total_completion_tokens
def _parse_token_id_placeholder(token: str) -> int | None:
"""Extract token ID from a 'token_id:N' placeholder string."""
if not token.startswith("token_id:"):
return None
try:
return int(token[len("token_id:") :])
except ValueError:
return None
def _correct_decoded_token(
token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike
) -> str:
"""Use preceding tokens as context to fix U+FFFD from byte-fallback.
Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py.
"""
max_ctx = min(len(context_token_ids), 4)
for num_ctx in range(1, max_ctx + 1):
context = context_token_ids[-num_ctx:]
full_decoded = tokenizer.decode(context + [token_id])
if full_decoded.endswith("�"):
continue
clean_end = len(context)
for j in range(len(context) - 1, -1, -1):
if tokenizer.decode([context[j]]).endswith("�"):
clean_end = j
else:
break
clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else ""
if full_decoded.startswith(clean_prefix):
return full_decoded[len(clean_prefix) :]
common_len = 0
for a, b in zip(clean_prefix, full_decoded):
if a != b:
break
common_len += 1
return full_decoded[common_len:]
return ""
def _resolve_logprobs(
logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
) -> ChatCompletionLogProbs:
"""Resolve token_id:N placeholders in a ChatCompletionLogProbs object."""
if logprobs.content is None:
return logprobs
context_token_ids: list[int] = []
resolved_content = []
for entry in logprobs.content:
token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
sampled_id = _parse_token_id_placeholder(entry.token)
if token_str.endswith("�") and sampled_id is not None:
token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer)
token_bytes = list(token_str.encode("utf-8"))
resolved_top = []
for top in entry.top_logprobs:
top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
top_id = _parse_token_id_placeholder(top.token)
if top_str.endswith("�") and top_id is not None:
top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer)
top_bytes = list(top_str.encode("utf-8"))
resolved_top.append(
top.model_copy(update={"token": top_str, "bytes": top_bytes})
)
resolved_content.append(
entry.model_copy(
update={
"token": token_str,
"bytes": token_bytes,
"top_logprobs": resolved_top,
}
)
)
if sampled_id is not None:
context_token_ids.append(sampled_id)
return ChatCompletionLogProbs(content=resolved_content)
def _convert_chat_logprobs_to_completion_logprobs(
logprobs: ChatCompletionLogProbs,
) -> CompletionLogProbs:
"""Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
(parallel flat lists) as required by the /v1/completions response schema."""
if logprobs.content is None:
return CompletionLogProbs()
tokens: list[str] = []
token_logprobs: list[float | None] = []
top_logprobs_list: list[dict[str, float] | None] = []
text_offset: list[int] = []
offset = 0
for entry in logprobs.content:
text_offset.append(offset)
tokens.append(entry.token)
token_logprobs.append(entry.logprob)
top_logprobs_list.append(
{t.token: t.logprob for t in entry.top_logprobs}
if entry.top_logprobs
else None
)
offset += len(entry.token)
return CompletionLogProbs(
text_offset=text_offset,
token_logprobs=token_logprobs,
tokens=tokens,
top_logprobs=top_logprobs_list,
)
+417
View File
@@ -0,0 +1,417 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from http import HTTPStatus
from typing import Any
from openai_harmony import Message as OpenAIMessage
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import (
ChatTemplateContentFormatOption,
ConversationMessage,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.openai.parser.harmony_utils import (
build_harmony_preamble,
extract_instructions_from_messages,
parse_chat_inputs_to_harmony_messages,
render_for_completion,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import (
EngineInput,
PromptType,
SingletonPrompt,
tokens_input,
)
from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs.preprocess import (
parse_model_prompt,
prompt_to_seq,
)
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
from vllm.utils.mistral import mt as _mt
logger = init_logger(__name__)
class OnlineRenderer:
def __init__(
self,
model_config: ModelConfig,
renderer: BaseRenderer,
*,
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
enable_auto_tools: bool = False,
exclude_tools_when_tool_choice_none: bool = False,
tool_parser: str | None = None,
reasoning_parser: str | None = None,
default_chat_template_kwargs: dict[str, Any] | None = None,
log_error_stack: bool = False,
) -> None:
self.model_config = model_config
self.renderer = renderer
self.request_logger = request_logger
self.enable_auto_tools = enable_auto_tools
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
self.parser: type[Parser] | None = ParserManager.get_parser(
tool_parser_name=tool_parser,
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
model_name=model_config.model,
is_harmony=self.use_harmony,
)
self.chat_template = chat_template
self.chat_template_content_format: ChatTemplateContentFormatOption = (
chat_template_content_format
)
self.default_chat_template_kwargs: dict[str, Any] = (
default_chat_template_kwargs or {}
)
self.trust_request_chat_template = trust_request_chat_template
self.log_error_stack = log_error_stack
self.supports_browsing = False
self.supports_code_interpreter = False
async def render_chat(
self,
request: ChatCompletionRequest,
*,
skip_mm_cache: bool = False,
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
"""Core preprocessing logic for chat requests (no model/engine check).
Called directly by render_chat_request and delegated to by
OpenAIServingChat.render_chat_request after its engine-aware checks.
"""
tokenizer = self.renderer.tokenizer
tool_parser = self.parser.tool_parser_cls if self.parser is not None else None
if is_mistral_tokenizer(tokenizer):
# because of issues with pydantic we need to potentially
# re-serialize the tool_calls field of the request
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
_mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
_mt.validate_request_params(request)
# Check if tool parsing is unavailable (common condition)
tool_parsing_unavailable = (
tool_parser is None
and not is_mistral_tokenizer(tokenizer)
and not self.use_harmony
)
# Validate tool_choice when tool parsing is required but unavailable
if tool_parsing_unavailable and request.tool_choice not in (
None,
"none",
):
if request.tool_choice == "auto" and not self.enable_auto_tools:
# for hf tokenizers, "auto" tools requires
# --enable-auto-tool-choice and --tool-call-parser
return self.create_error_response(
'"auto" tool choice requires '
"--enable-auto-tool-choice and --tool-call-parser to be set"
)
elif request.tool_choice != "auto":
# "required" or named tool requires tool parser
return self.create_error_response(
f'tool_choice="{request.tool_choice}" requires '
"--tool-call-parser to be set"
)
if request.tools is None or (
request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
):
tool_dicts = None
else:
tool_dicts = [tool.model_dump() for tool in request.tools]
if not self.use_harmony:
# Common case.
error_check_ret = self.validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if error_check_ret is not None:
return error_check_ret
conversation, engine_inputs = await self.preprocess_chat(
request,
request.messages,
default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=self.default_chat_template_kwargs,
tool_dicts=tool_dicts,
parser=self.parser,
skip_mm_cache=skip_mm_cache,
)
else:
# For GPT-OSS.
should_include_tools = tool_dicts is not None
conversation, engine_inputs = self._make_request_with_harmony(
request, should_include_tools
)
return conversation, engine_inputs
def _make_request_with_harmony(
self,
request: ChatCompletionRequest,
should_include_tools: bool = True,
):
"""Build Harmony (GPT-OSS) messages and engine prompt from a chat request."""
messages: list[OpenAIMessage] = []
# because of issues with pydantic we need to potentially
# re-serialize the tool_calls field of the request
# for more info: see comment in `maybe_serialize_tool_calls`
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
chat_messages = list(request.messages)
instructions, chat_messages = extract_instructions_from_messages(chat_messages)
# Add system message.
# NOTE: In Chat Completion API, browsing is enabled by default
# if the model supports it. TODO: Support browsing.
assert not self.supports_browsing
assert not self.supports_code_interpreter
if (reasoning_effort := request.reasoning_effort) == "none":
raise ValueError(f"Harmony does not support {reasoning_effort=}")
tools = request.tools if should_include_tools else None
messages.extend(
build_harmony_preamble(
instructions=instructions,
tools=tools, # type: ignore[arg-type]
reasoning_effort=reasoning_effort,
with_custom_tools=should_include_tools,
)
)
# Add remaining conversation messages.
messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages))
# Render prompt token ids.
prompt_token_ids = render_for_completion(messages)
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
return messages, [engine_input]
async def render_completion(
self,
request: CompletionRequest,
*,
skip_mm_cache: bool = False,
) -> list[EngineInput] | ErrorResponse:
"""Core preprocessing logic for completion requests (no model/engine check).
Called directly by render_completion_request and delegated to by
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
"""
# Return error for unsupported features.
if request.suffix is not None:
return self.create_error_response("suffix is not currently supported")
if request.echo and request.prompt_embeds is not None:
return self.create_error_response("Echo is unsupported with prompt embeds.")
if request.prompt_logprobs is not None and request.prompt_embeds is not None:
return self.create_error_response(
"prompt_logprobs is not compatible with prompt embeds."
)
engine_inputs = await self.preprocess_completion(
request,
prompt_input=request.prompt,
prompt_embeds=request.prompt_embeds,
skip_mm_cache=skip_mm_cache,
)
return engine_inputs
def create_error_response(
self,
message: str | Exception,
err_type: str = "BadRequestError",
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
param: str | None = None,
) -> ErrorResponse:
return create_error_response(message, err_type, status_code, param)
def validate_chat_template(
self,
request_chat_template: str | None,
chat_template_kwargs: dict[str, Any] | None,
trust_request_chat_template: bool,
) -> ErrorResponse | None:
"""Copied from OpenAIServing._validate_chat_template."""
if not trust_request_chat_template and (
request_chat_template is not None
or (
chat_template_kwargs
and chat_template_kwargs.get("chat_template") is not None
)
):
return self.create_error_response(
"Chat template is passed with request, but "
"--trust-request-chat-template is not set. "
"Refused request with untrusted chat template."
)
return None
async def preprocess_completion(
self,
request: Any,
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
prompt_embeds: bytes | list[bytes] | None,
*,
skip_mm_cache: bool = False,
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_completion."""
prompts = list[SingletonPrompt | bytes]()
if prompt_embeds is not None: # embeds take higher priority
prompts.extend(prompt_to_seq(prompt_embeds))
if prompt_input is not None:
prompts.extend(prompt_to_seq(prompt_input))
return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache)
async def preprocess_cmpl(
self,
request: Any,
prompts: Sequence[PromptType | bytes],
*,
skip_mm_cache: bool = False,
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_cmpl."""
renderer = self.renderer
model_config = self.model_config
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(model_config, prompt)
)
for prompt in prompts
]
tok_params = request.build_tok_params(model_config)
return await renderer.render_cmpl_async(
parsed_prompts,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
skip_mm_cache=skip_mm_cache,
)
async def preprocess_chat(
self,
request: Any,
messages: list[Any],
default_template: str | None,
default_template_content_format: ChatTemplateContentFormatOption,
default_template_kwargs: dict[str, Any] | None,
tool_dicts: list[dict[str, Any]] | None = None,
parser: type[Parser] | None = None,
*,
skip_mm_cache: bool = False,
) -> tuple[list[ConversationMessage], list[EngineInput]]:
"""Copied from OpenAIServing._preprocess_chat."""
renderer = self.renderer
mm_config = self.model_config.multimodal_config
default_template_kwargs = merge_kwargs(
default_template_kwargs,
dict(
tools=tool_dicts,
tokenize=(
is_mistral_tokenizer(renderer.tokenizer)
or self.model_config.enable_prompt_embeds
),
),
)
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
default_template, default_template_content_format
).with_defaults(
default_template_kwargs,
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
)
(conversation,), (engine_input,) = await renderer.render_chat_async(
[messages],
chat_params,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
skip_mm_cache=skip_mm_cache,
)
# tool parsing is done only if a tool_parser has been set and if
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
# is set, we want to prevent parsing a tool_call hallucinated by the LLM
#
# Exception: Mistral grammar-capable tokenizers always call
# adjust_request — even for tool_choice="none" — so that the grammar
# factory can prevent special-token leakage.
if parser is not None:
tokenizer = renderer.get_tokenizer()
tool_parser = parser.tool_parser_cls
tool_choice = getattr(request, "tool_choice", "none")
is_mistral_grammar_eligible = (
tool_parser is not None
and is_mistral_tool_parser(tool_parser)
and is_mistral_tokenizer(tokenizer)
and tokenizer.supports_grammar
)
should_adjust_request = (
parser.reasoning_parser_cls is not None
or tool_choice != "none"
or is_mistral_grammar_eligible
)
if should_adjust_request:
if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
msg = (
"Tool usage is only supported "
"for Chat Completions API or Responses API requests, "
f"but got {type(request).__name__}"
)
raise NotImplementedError(msg)
request = parser(
tokenizer,
request.tools,
model_config=self.model_config,
chat_template_kwargs=chat_params.chat_template_kwargs,
).adjust_request(
request=request,
)
return conversation, [engine_input]