[Frontend] Split ServingRender into renderer and entrypoint. (#44285)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -17,9 +17,11 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
@@ -88,19 +90,19 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
serving_chat = OpenAIServingChat(
|
||||
engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=serving_render,
|
||||
online_renderer=online_renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -113,7 +115,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
|
||||
serving_chat.openai_serving_render.preprocess_chat = AsyncMock(
|
||||
serving_chat.online_renderer.preprocess_chat = AsyncMock(
|
||||
side_effect=_fake_preprocess_chat
|
||||
)
|
||||
return serving_chat
|
||||
@@ -187,13 +189,46 @@ async def test_openai_chat_keeps_mm_cache_for_engine_execution():
|
||||
|
||||
assert isinstance(result, tuple)
|
||||
assert (
|
||||
serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
serving_chat.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_render(engine: AsyncLLM) -> ServingRender:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
online_derenderer = OnlineDerenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
serving_render = ServingRender(models, online_renderer, online_derenderer)
|
||||
|
||||
async def _fake_preprocess_chat(*args, **kwargs):
|
||||
# return conversation, engine_inputs
|
||||
return (
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
|
||||
serving_render.online_renderer.preprocess_chat = AsyncMock(
|
||||
side_effect=_fake_preprocess_chat
|
||||
)
|
||||
return serving_render
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_renderer_only_chat_request_skips_mm_cache():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
@@ -202,20 +237,18 @@ async def test_renderer_only_chat_request_skips_mm_cache():
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_chat = _build_serving_chat(mock_engine)
|
||||
serving_render = _build_serving_render(mock_engine)
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model=MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
)
|
||||
|
||||
result = await serving_chat.openai_serving_render.render_chat_request(request)
|
||||
result = await serving_render.render_chat_request(request)
|
||||
|
||||
assert result.token_ids == [1, 2, 3]
|
||||
assert (
|
||||
serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
serving_render.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@@ -38,7 +38,6 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
@@ -46,6 +45,7 @@ from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.parser import HarmonyParser
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.mistral import MistralRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
@@ -567,13 +567,12 @@ def _build_renderer(model_config: MockModelConfig):
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_render(
|
||||
def _build_online_renderer(
|
||||
engine, model_registry: OpenAIModelRegistry
|
||||
) -> OpenAIServingRender:
|
||||
return OpenAIServingRender(
|
||||
) -> OnlineRenderer:
|
||||
return OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=model_registry,
|
||||
request_logger=None,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
@@ -591,13 +590,13 @@ def _build_serving_chat(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
openai_serving_render = _build_serving_render(engine, models.registry)
|
||||
online_renderer = _build_online_renderer(engine, models.registry)
|
||||
|
||||
serving_chat = OpenAIServingChat(
|
||||
engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=openai_serving_render,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
@@ -621,13 +620,13 @@ async def _async_serving_chat_init():
|
||||
engine = MockEngine()
|
||||
|
||||
models = OpenAIServingModels(engine, BASE_MODEL_PATHS)
|
||||
openai_serving_render = _build_serving_render(engine, models.registry)
|
||||
online_renderer = _build_online_renderer(engine, models.registry)
|
||||
|
||||
serving_completion = OpenAIServingChat(
|
||||
engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=openai_serving_render,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
@@ -1396,9 +1395,7 @@ class TestServingChatWithHarmony:
|
||||
messages=messages,
|
||||
include_reasoning=include_reasoning,
|
||||
)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
[
|
||||
@@ -1429,8 +1426,8 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the second turn's input
|
||||
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
input_messages_2, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
|
||||
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_2
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_2,
|
||||
@@ -1452,9 +1449,7 @@ class TestServingChatWithHarmony:
|
||||
{"role": "user", "content": "Hello"},
|
||||
]
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
[
|
||||
@@ -1478,9 +1473,7 @@ class TestServingChatWithHarmony:
|
||||
req = ChatCompletionRequest(
|
||||
model=MODEL_NAME, messages=messages, tools=weather_tools
|
||||
)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
[
|
||||
@@ -1503,9 +1496,7 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the first turn's input
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
[
|
||||
@@ -1549,8 +1540,8 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the second turn's input
|
||||
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages_2, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
|
||||
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_2
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_2,
|
||||
@@ -1588,9 +1579,7 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the first turn's input
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
[
|
||||
@@ -1634,8 +1623,8 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the second turn's input
|
||||
req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages_2, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req_2)
|
||||
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_2
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_2,
|
||||
@@ -1686,8 +1675,8 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the third turn's input
|
||||
req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages_3, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req_3)
|
||||
input_messages_3, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_3
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_3,
|
||||
@@ -1751,8 +1740,8 @@ class TestServingChatWithHarmony:
|
||||
|
||||
# Test the Harmony messages for the fourth turn's input
|
||||
req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
|
||||
input_messages_4, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req_4)
|
||||
input_messages_4, _ = serving_chat.online_renderer._make_request_with_harmony(
|
||||
req_4
|
||||
)
|
||||
verify_harmony_messages(
|
||||
input_messages_4,
|
||||
@@ -1802,9 +1791,7 @@ class TestServingChatWithHarmony:
|
||||
},
|
||||
]
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
@@ -1835,9 +1822,7 @@ class TestServingChatWithHarmony:
|
||||
},
|
||||
]
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
@@ -1866,9 +1851,7 @@ class TestServingChatWithHarmony:
|
||||
},
|
||||
]
|
||||
req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
|
||||
input_messages, _ = (
|
||||
serving_chat.openai_serving_render._make_request_with_harmony(req)
|
||||
)
|
||||
input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req)
|
||||
|
||||
verify_harmony_messages(
|
||||
input_messages,
|
||||
@@ -1898,14 +1881,14 @@ async def test_tool_choice_validation_without_parser():
|
||||
engine_client=mock_engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
openai_serving_render = _build_serving_render(mock_engine, models.registry)
|
||||
online_renderer = _build_online_renderer(mock_engine, models.registry)
|
||||
|
||||
# Create serving_chat without tool_parser (enable_auto_tools=False)
|
||||
serving_chat = OpenAIServingChat(
|
||||
mock_engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=openai_serving_render,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
@@ -1967,13 +1950,13 @@ async def test_streaming_n_gt1_independent_tool_parsers():
|
||||
engine_client=mock_engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
openai_serving_render = _build_serving_render(mock_engine, models.registry)
|
||||
online_renderer = _build_online_renderer(mock_engine, models.registry)
|
||||
|
||||
serving_chat = OpenAIServingChat(
|
||||
mock_engine,
|
||||
models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=openai_serving_render,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=CHAT_TEMPLATE,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
|
||||
@@ -14,9 +14,11 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
@@ -77,10 +79,9 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -88,7 +89,7 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion:
|
||||
return OpenAIServingCompletion(
|
||||
engine,
|
||||
models,
|
||||
openai_serving_render=serving_render,
|
||||
online_renderer=online_renderer,
|
||||
request_logger=None,
|
||||
)
|
||||
|
||||
@@ -158,7 +159,7 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_completion = _build_serving_completion(mock_engine)
|
||||
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
|
||||
serving_completion.online_renderer.preprocess_completion = AsyncMock(
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
@@ -171,13 +172,48 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution():
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert (
|
||||
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
|
||||
serving_completion.online_renderer.preprocess_completion.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_render(engine: AsyncLLM) -> ServingRender:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
online_derenderer = OnlineDerenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
serving_render = ServingRender(models, online_renderer, online_derenderer)
|
||||
|
||||
async def _fake_preprocess_chat(*args, **kwargs):
|
||||
# return conversation, engine_inputs
|
||||
return (
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
)
|
||||
|
||||
serving_render.online_renderer.preprocess_chat = AsyncMock(
|
||||
side_effect=_fake_preprocess_chat
|
||||
)
|
||||
return serving_render
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_renderer_only_completion_request_skips_mm_cache():
|
||||
mock_engine = MagicMock(spec=AsyncLLM)
|
||||
@@ -186,8 +222,9 @@ async def test_renderer_only_completion_request_skips_mm_cache():
|
||||
mock_engine.input_processor = MagicMock()
|
||||
mock_engine.renderer = _build_renderer(mock_engine.model_config)
|
||||
|
||||
serving_completion = _build_serving_completion(mock_engine)
|
||||
serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
|
||||
serving_render = _build_serving_render(mock_engine)
|
||||
|
||||
serving_render.online_renderer.preprocess_completion = AsyncMock(
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
@@ -196,13 +233,11 @@ async def test_renderer_only_completion_request_skips_mm_cache():
|
||||
prompt="Test prompt",
|
||||
)
|
||||
|
||||
result = await serving_completion.openai_serving_render.render_completion_request(
|
||||
request
|
||||
)
|
||||
result = await serving_render.render_completion_request(request)
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert (
|
||||
serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
|
||||
serving_render.online_renderer.preprocess_completion.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
is True
|
||||
|
||||
@@ -14,10 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
@@ -144,16 +144,15 @@ def mock_serving_setup():
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=mock_engine.model_config,
|
||||
renderer=mock_engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
serving_completion = OpenAIServingCompletion(
|
||||
mock_engine, models, openai_serving_render=serving_render, request_logger=None
|
||||
mock_engine, models, online_renderer=online_renderer, request_logger=None
|
||||
)
|
||||
|
||||
return mock_engine, serving_completion
|
||||
|
||||
@@ -230,7 +230,7 @@ class TestInitializeToolSessions:
|
||||
instance = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
openai_serving_render=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -316,7 +316,7 @@ class TestValidateGeneratorInput:
|
||||
instance = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
openai_serving_render=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -379,7 +379,7 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
|
||||
serving = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
openai_serving_render=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -684,7 +684,7 @@ def _make_serving_instance_with_reasoning():
|
||||
serving = OpenAIServingResponses(
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
openai_serving_render=MagicMock(),
|
||||
online_renderer=MagicMock(),
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
|
||||
@@ -428,8 +428,8 @@ class TestPreProcessCohereOnline:
|
||||
handler._get_task_instruction_prefix = lambda _input_type: None
|
||||
handler._has_chat_template = lambda: False
|
||||
handler._preprocess_cmpl_online = preprocess_cmpl_online
|
||||
handler._batch_render_chat = lambda *_args, **_kwargs: (
|
||||
pytest.fail("text-only request should not require chat rendering")
|
||||
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
|
||||
"text-only request should not require chat rendering"
|
||||
)
|
||||
|
||||
handler._pre_process_cohere_online(ctx)
|
||||
@@ -448,8 +448,8 @@ class TestPreProcessCohereOnline:
|
||||
|
||||
handler._get_task_instruction_prefix = lambda _input_type: "query: "
|
||||
handler._has_chat_template = lambda: False
|
||||
handler._batch_render_chat = lambda *_args, **_kwargs: (
|
||||
pytest.fail("chat rendering should be skipped without a template")
|
||||
handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail(
|
||||
"chat rendering should be skipped without a template"
|
||||
)
|
||||
handler._preprocess_cmpl_online = preprocess_cmpl
|
||||
|
||||
@@ -485,8 +485,8 @@ class TestPreProcessCohereOnline:
|
||||
handler._get_task_instruction_prefix = lambda _input_type: "query: "
|
||||
handler._has_chat_template = lambda: True
|
||||
handler._batch_render_chat = batch_render_chat
|
||||
handler._preprocess_cmpl_online = lambda *_args, **_kwargs: (
|
||||
pytest.fail("completion path should be skipped when a template exists")
|
||||
handler._preprocess_cmpl_online = lambda *_args, **_kwargs: pytest.fail(
|
||||
"completion path should be skipped when a template exists"
|
||||
)
|
||||
|
||||
handler._pre_process_cohere_online(ctx)
|
||||
|
||||
@@ -17,10 +17,10 @@ from vllm.entrypoints.serve.disagg.protocol import (
|
||||
GenerateResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
@@ -92,10 +92,9 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -103,7 +102,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
|
||||
serving = ServingTokens(
|
||||
engine,
|
||||
models,
|
||||
openai_serving_render=serving_render,
|
||||
online_renderer=online_renderer,
|
||||
request_logger=None,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -111,7 +110,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
|
||||
async def _fake_preprocess(*args, **kwargs):
|
||||
return [{"prompt_token_ids": [1, 2, 3]}]
|
||||
|
||||
serving.openai_serving_render.preprocess_completion = AsyncMock(
|
||||
serving.online_renderer.preprocess_completion = AsyncMock(
|
||||
side_effect=_fake_preprocess
|
||||
)
|
||||
return serving
|
||||
@@ -199,9 +198,7 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert (
|
||||
serving.openai_serving_render.preprocess_completion.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@@ -10,12 +10,12 @@ import pytest
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
TokenizeChatRequest,
|
||||
TokenizeCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
@@ -63,18 +63,16 @@ def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization:
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
return ServingTokenization(
|
||||
models,
|
||||
openai_serving_render=serving_render,
|
||||
request_logger=None,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
@@ -89,7 +87,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
|
||||
mock_engine.renderer = MagicMock()
|
||||
|
||||
serving = _build_serving_tokenization(mock_engine)
|
||||
serving.openai_serving_render.preprocess_chat = AsyncMock(
|
||||
serving.online_renderer.preprocess_chat = AsyncMock(
|
||||
return_value=(
|
||||
[{"role": "user", "content": "Test"}],
|
||||
[{"prompt_token_ids": [1, 2, 3]}],
|
||||
@@ -105,7 +103,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
|
||||
|
||||
assert response.tokens == [1, 2, 3]
|
||||
assert (
|
||||
serving.openai_serving_render.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
@@ -119,7 +117,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
|
||||
mock_engine.renderer = MagicMock()
|
||||
|
||||
serving = _build_serving_tokenization(mock_engine)
|
||||
serving.openai_serving_render.preprocess_completion = AsyncMock(
|
||||
serving.online_renderer.preprocess_completion = AsyncMock(
|
||||
return_value=[{"prompt_token_ids": [1, 2, 3]}]
|
||||
)
|
||||
|
||||
@@ -132,8 +130,6 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
|
||||
|
||||
assert response.tokens == [1, 2, 3]
|
||||
assert (
|
||||
serving.openai_serving_render.preprocess_completion.call_args.kwargs[
|
||||
"skip_mm_cache"
|
||||
]
|
||||
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
@@ -99,8 +99,8 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
|
||||
engine_client = SimpleNamespace(
|
||||
errored=False,
|
||||
generate=Mock(
|
||||
side_effect=lambda *_args, **_kwargs: (
|
||||
_records_start_then_never_finishes(started_request_ids, _args[2])
|
||||
side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes(
|
||||
started_request_ids, _args[2]
|
||||
)
|
||||
),
|
||||
abort=AsyncMock(),
|
||||
|
||||
@@ -512,12 +512,11 @@ async def test_header_dp_rank_argument():
|
||||
)
|
||||
|
||||
# Create render serving instance (required by OpenAIServingChat)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
serving_render = OpenAIServingRender(
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
@@ -528,7 +527,7 @@ async def test_header_dp_rank_argument():
|
||||
engine_client=engine,
|
||||
models=models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=serving_render,
|
||||
online_renderer=online_renderer,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
|
||||
@@ -10,7 +10,7 @@ import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import Any
|
||||
|
||||
import jinja2
|
||||
from fastapi import Request
|
||||
@@ -48,9 +48,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -111,7 +109,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
models: OpenAIServingModels,
|
||||
response_role: str,
|
||||
*,
|
||||
openai_serving_render: "OpenAIServingRender",
|
||||
online_renderer: "OnlineRenderer",
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
@@ -127,7 +125,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
engine_client=engine_client,
|
||||
models=models,
|
||||
response_role=response_role,
|
||||
openai_serving_render=openai_serving_render,
|
||||
online_renderer=online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=chat_template_content_format,
|
||||
|
||||
@@ -87,7 +87,7 @@ async def init_generate_state(
|
||||
tool_server = None
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
# Render endpoints are always backed by OpenAIServingRender so that
|
||||
# Render endpoints are always backed by OnlineRenderer so that
|
||||
# /v1/chat/completions/render and /v1/completions/render work on both
|
||||
# generate-mode and render-only servers. Created in init_app_state.
|
||||
|
||||
@@ -95,7 +95,7 @@ async def init_generate_state(
|
||||
OpenAIServingResponses(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
state.openai_serving_render,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
@@ -116,7 +116,7 @@ async def init_generate_state(
|
||||
engine_client=engine_client,
|
||||
models=state.openai_serving_models,
|
||||
response_role=args.response_role,
|
||||
openai_serving_render=state.openai_serving_render,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
@@ -146,7 +146,7 @@ async def init_generate_state(
|
||||
OpenAIServingCompletion(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
openai_serving_render=state.openai_serving_render,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
@@ -160,7 +160,7 @@ async def init_generate_state(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
args.response_role,
|
||||
openai_serving_render=state.openai_serving_render,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
@@ -179,7 +179,7 @@ async def init_generate_state(
|
||||
ServingTokens(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
state.openai_serving_render,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
|
||||
@@ -32,7 +32,7 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
@@ -55,6 +55,8 @@ from vllm.entrypoints.serve.utils.server_utils import (
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tracing import instrument
|
||||
@@ -360,10 +362,24 @@ async def init_app_state(
|
||||
)
|
||||
await state.openai_serving_models.init_static_loras()
|
||||
|
||||
state.openai_serving_render = OpenAIServingRender(
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
model_registry=state.openai_serving_models.registry,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
@@ -378,13 +394,19 @@ async def init_app_state(
|
||||
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
state.openai_serving_models,
|
||||
state.openai_serving_render,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
state.serving_render = ServingRender(
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
state.online_derenderer,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||
@@ -423,8 +445,8 @@ async def init_render_app_state(
|
||||
"""
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
served_model_names = args.served_model_name or [args.model]
|
||||
model_registry = OpenAIModelRegistry(
|
||||
@@ -443,10 +465,24 @@ async def init_render_app_state(
|
||||
renderer = renderer_from_config(vllm_config)
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
state.openai_serving_render = OpenAIServingRender(
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
state.online_derenderer = OnlineDerenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
renderer=renderer,
|
||||
model_registry=model_registry,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
@@ -462,13 +498,19 @@ async def init_render_app_state(
|
||||
state.openai_serving_models = model_registry
|
||||
state.serving_tokenization = ServingTokenization(
|
||||
model_registry,
|
||||
state.openai_serving_render,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
state.serving_render = ServingRender(
|
||||
model_registry,
|
||||
state.online_renderer,
|
||||
state.online_derenderer,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
state.vllm_config = vllm_config
|
||||
# Disable stats logging — there is no engine to poll.
|
||||
|
||||
@@ -47,7 +47,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
"""Validate the model and preprocess a batched chat completion request.
|
||||
|
||||
Performs engine-aware checks then delegates per-conversation
|
||||
preprocessing to OpenAIServingRender, validating the chat template
|
||||
preprocessing to OnlineRenderer, validating the chat template
|
||||
once for the whole batch.
|
||||
|
||||
Returns:
|
||||
@@ -62,19 +62,19 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
|
||||
render = self.openai_serving_render
|
||||
renderer = self.online_renderer
|
||||
|
||||
if not render.use_harmony:
|
||||
if not renderer.use_harmony:
|
||||
# Common case: validate the chat template once for the whole batch.
|
||||
error_check_ret = render.validate_chat_template(
|
||||
error_check_ret = renderer.validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=render.trust_request_chat_template,
|
||||
trust_request_chat_template=renderer.trust_request_chat_template,
|
||||
)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
parser = render.parser
|
||||
parser = renderer.parser
|
||||
tool_dicts: list[dict] | None = None
|
||||
|
||||
all_conversations: list[list[ConversationMessage]] = []
|
||||
@@ -82,17 +82,17 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
|
||||
for messages in request.messages:
|
||||
single_request = request.to_chat_completion_request(messages)
|
||||
if render.use_harmony:
|
||||
conversation, engine_prompts = render._make_request_with_harmony(
|
||||
if renderer.use_harmony:
|
||||
conversation, engine_prompts = renderer._make_request_with_harmony(
|
||||
single_request, should_include_tools=tool_dicts is not None
|
||||
)
|
||||
else:
|
||||
conversation, engine_prompts = await render.preprocess_chat(
|
||||
conversation, engine_prompts = await renderer.preprocess_chat(
|
||||
single_request,
|
||||
messages,
|
||||
default_template=render.chat_template,
|
||||
default_template_content_format=render.chat_template_content_format,
|
||||
default_template_kwargs=render.default_chat_template_kwargs,
|
||||
default_template=renderer.chat_template,
|
||||
default_template_content_format=renderer.chat_template_content_format,
|
||||
default_template_kwargs=renderer.default_chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
parser=parser,
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import Any, Final, cast
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
@@ -61,14 +61,12 @@ from vllm.outputs import RequestOutput
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.renderers import ChatParams
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.mistral import is_mistral_tool_parser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -112,7 +110,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
models: OpenAIServingModels,
|
||||
response_role: str,
|
||||
*,
|
||||
openai_serving_render: "OpenAIServingRender",
|
||||
online_renderer: "OnlineRenderer",
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
@@ -135,7 +133,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
)
|
||||
|
||||
self.openai_serving_render = openai_serving_render
|
||||
self.online_renderer = online_renderer
|
||||
self.response_role = response_role
|
||||
self.chat_template = chat_template
|
||||
self.chat_template_content_format: Final = chat_template_content_format
|
||||
@@ -212,7 +210,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
"""
|
||||
Validate the model and preprocess a chat completion request.
|
||||
|
||||
Delegates preprocessing logic to OpenAIServingRender, adding the
|
||||
Delegates preprocessing logic to OnlineRenderer, adding the
|
||||
engine-aware checks (LoRA model validation, engine health).
|
||||
|
||||
Returns:
|
||||
@@ -230,7 +228,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
|
||||
return await self.openai_serving_render.render_chat(request)
|
||||
return await self.online_renderer.render_chat(request)
|
||||
|
||||
async def create_chat_completion(
|
||||
self,
|
||||
|
||||
@@ -6,7 +6,7 @@ import io
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
from typing import TYPE_CHECKING, cast
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
@@ -41,14 +41,12 @@ from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
from vllm.utils.collection_utils import as_list
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -58,7 +56,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
openai_serving_render: "OpenAIServingRender",
|
||||
online_renderer: "OnlineRenderer",
|
||||
request_logger: RequestLogger | None,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
@@ -71,7 +69,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
)
|
||||
|
||||
self.openai_serving_render = openai_serving_render
|
||||
self.online_renderer = online_renderer
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
self.enable_force_include_usage = enable_force_include_usage
|
||||
|
||||
@@ -90,7 +88,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
"""
|
||||
Validate the model and preprocess a completion request.
|
||||
|
||||
Delegates preprocessing logic to OpenAIServingRender, adding the
|
||||
Delegates preprocessing logic to OnlineRenderer, adding the
|
||||
engine-aware checks (LoRA model validation, engine health).
|
||||
|
||||
Returns:
|
||||
@@ -106,7 +104,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
|
||||
return await self.openai_serving_render.render_completion(request)
|
||||
return await self.online_renderer.render_completion(request)
|
||||
|
||||
async def create_completion(
|
||||
self,
|
||||
|
||||
@@ -13,12 +13,8 @@ from starlette.datastructures import Headers
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
GenerationError,
|
||||
|
||||
@@ -92,7 +92,6 @@ from vllm.entrypoints.openai.responses.utils import (
|
||||
extract_function_tool_names,
|
||||
extract_tool_types,
|
||||
)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
@@ -103,6 +102,7 @@ from vllm.logprobs import SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import random_uuid
|
||||
@@ -155,7 +155,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
openai_serving_render: OpenAIServingRender,
|
||||
online_renderer: OnlineRenderer,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
@@ -177,7 +177,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
)
|
||||
|
||||
self.openai_serving_render = openai_serving_render
|
||||
self.online_renderer = online_renderer
|
||||
self.chat_template = chat_template
|
||||
self.chat_template_content_format: Final = chat_template_content_format
|
||||
self.chat_template_kwargs = default_chat_template_kwargs or {}
|
||||
@@ -633,7 +633,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
prev_response_output=prev_response.output if prev_response else None,
|
||||
)
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
||||
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
|
||||
_, engine_inputs = await self.online_renderer.preprocess_chat(
|
||||
request,
|
||||
messages,
|
||||
default_template=self.chat_template,
|
||||
@@ -657,7 +657,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
request_input=messages,
|
||||
)
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
||||
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
|
||||
_, engine_inputs = await self.online_renderer.preprocess_chat(
|
||||
request,
|
||||
new_messages,
|
||||
default_template=chat_template,
|
||||
|
||||
@@ -10,9 +10,7 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Respons
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.disagg.protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
|
||||
@@ -36,7 +36,6 @@ from vllm.entrypoints.serve.disagg.protocol import (
|
||||
GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import EngineInput, mm_input
|
||||
@@ -48,6 +47,7 @@ from vllm.multimodal.inputs import (
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.utils.collection_utils import as_list
|
||||
|
||||
@@ -61,7 +61,7 @@ class ServingTokens(OpenAIServing):
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
openai_serving_render: OpenAIServingRender,
|
||||
online_renderer: OnlineRenderer,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
force_no_detokenize: bool = False,
|
||||
@@ -75,7 +75,7 @@ class ServingTokens(OpenAIServing):
|
||||
request_logger=request_logger,
|
||||
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
||||
)
|
||||
self.openai_serving_render = openai_serving_render
|
||||
self.online_renderer = online_renderer
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
self.enable_log_outputs = enable_log_outputs
|
||||
self.force_no_detokenize = force_no_detokenize
|
||||
@@ -168,7 +168,7 @@ class ServingTokens(OpenAIServing):
|
||||
cache_salt=request.cache_salt,
|
||||
)
|
||||
else:
|
||||
(engine_input,) = await self.openai_serving_render.preprocess_completion(
|
||||
(engine_input,) = await self.online_renderer.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.token_ids,
|
||||
prompt_embeds=None,
|
||||
|
||||
@@ -15,7 +15,12 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
|
||||
from vllm.entrypoints.serve.disagg.protocol import (
|
||||
DerenderChatRequest,
|
||||
DerenderCompletionRequest,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
DetokenizeRequest,
|
||||
TokenizeChatRequest,
|
||||
@@ -45,11 +50,17 @@ class RendererChatRequest(RendererRequest, Protocol):
|
||||
|
||||
|
||||
CompletionLikeRequest: TypeAlias = (
|
||||
CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest
|
||||
CompletionRequest
|
||||
| TokenizeCompletionRequest
|
||||
| DetokenizeRequest
|
||||
| DerenderCompletionRequest
|
||||
)
|
||||
|
||||
ChatLikeRequest: TypeAlias = (
|
||||
ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest
|
||||
ChatCompletionRequest
|
||||
| BatchChatCompletionRequest
|
||||
| TokenizeChatRequest
|
||||
| DerenderChatRequest
|
||||
)
|
||||
|
||||
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
|
||||
DerenderCompletionRequest,
|
||||
GenerateRequest,
|
||||
)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.render.serving import ServingRender
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -28,8 +28,8 @@ logger = init_logger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def render(request: Request) -> OpenAIServingRender | None:
|
||||
return getattr(request.app.state, "openai_serving_render", None)
|
||||
def render(request: Request) -> ServingRender | None:
|
||||
return getattr(request.app.state, "serving_render", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
|
||||
@@ -1,44 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from http import HTTPStatus
|
||||
from typing import Any, cast
|
||||
from typing import cast
|
||||
|
||||
from openai_harmony import Message as OpenAIMessage
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionResponseChoice,
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionLogProbs,
|
||||
CompletionRequest,
|
||||
CompletionResponse,
|
||||
CompletionResponseChoice,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
ToolCall,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
build_harmony_preamble,
|
||||
extract_instructions_from_messages,
|
||||
parse_chat_inputs_to_harmony_messages,
|
||||
render_for_completion,
|
||||
from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
|
||||
from vllm.entrypoints.serve.disagg.protocol import (
|
||||
DerenderChatRequest,
|
||||
@@ -47,207 +27,49 @@ from vllm.entrypoints.serve.disagg.protocol import (
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import (
|
||||
EngineInput,
|
||||
MultiModalHashes,
|
||||
MultiModalInput,
|
||||
MultiModalPlaceholders,
|
||||
PromptType,
|
||||
SingletonPrompt,
|
||||
tokens_input,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
extract_prompt_len,
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
|
||||
from vllm.utils.mistral import mt as _mt
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _parse_token_id_placeholder(token: str) -> int | None:
|
||||
"""Extract token ID from a 'token_id:N' placeholder string."""
|
||||
if not token.startswith("token_id:"):
|
||||
return None
|
||||
try:
|
||||
return int(token[len("token_id:") :])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _correct_decoded_token(
|
||||
token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike
|
||||
) -> str:
|
||||
"""Use preceding tokens as context to fix U+FFFD from byte-fallback.
|
||||
|
||||
Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py.
|
||||
"""
|
||||
max_ctx = min(len(context_token_ids), 4)
|
||||
|
||||
for num_ctx in range(1, max_ctx + 1):
|
||||
context = context_token_ids[-num_ctx:]
|
||||
full_decoded = tokenizer.decode(context + [token_id])
|
||||
|
||||
if full_decoded.endswith("�"):
|
||||
continue
|
||||
|
||||
clean_end = len(context)
|
||||
for j in range(len(context) - 1, -1, -1):
|
||||
if tokenizer.decode([context[j]]).endswith("�"):
|
||||
clean_end = j
|
||||
else:
|
||||
break
|
||||
|
||||
clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else ""
|
||||
|
||||
if full_decoded.startswith(clean_prefix):
|
||||
return full_decoded[len(clean_prefix) :]
|
||||
|
||||
common_len = 0
|
||||
for a, b in zip(clean_prefix, full_decoded):
|
||||
if a != b:
|
||||
break
|
||||
common_len += 1
|
||||
return full_decoded[common_len:]
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def _resolve_logprobs(
|
||||
logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Resolve token_id:N placeholders in a ChatCompletionLogProbs object."""
|
||||
if logprobs.content is None:
|
||||
return logprobs
|
||||
|
||||
context_token_ids: list[int] = []
|
||||
resolved_content = []
|
||||
|
||||
for entry in logprobs.content:
|
||||
token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
|
||||
sampled_id = _parse_token_id_placeholder(entry.token)
|
||||
|
||||
if token_str.endswith("�") and sampled_id is not None:
|
||||
token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer)
|
||||
token_bytes = list(token_str.encode("utf-8"))
|
||||
|
||||
resolved_top = []
|
||||
for top in entry.top_logprobs:
|
||||
top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
|
||||
top_id = _parse_token_id_placeholder(top.token)
|
||||
if top_str.endswith("�") and top_id is not None:
|
||||
top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer)
|
||||
top_bytes = list(top_str.encode("utf-8"))
|
||||
resolved_top.append(
|
||||
top.model_copy(update={"token": top_str, "bytes": top_bytes})
|
||||
)
|
||||
|
||||
resolved_content.append(
|
||||
entry.model_copy(
|
||||
update={
|
||||
"token": token_str,
|
||||
"bytes": token_bytes,
|
||||
"top_logprobs": resolved_top,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
if sampled_id is not None:
|
||||
context_token_ids.append(sampled_id)
|
||||
|
||||
return ChatCompletionLogProbs(content=resolved_content)
|
||||
|
||||
|
||||
def _convert_chat_logprobs_to_completion_logprobs(
|
||||
logprobs: ChatCompletionLogProbs,
|
||||
) -> CompletionLogProbs:
|
||||
"""Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
|
||||
(parallel flat lists) as required by the /v1/completions response schema."""
|
||||
if logprobs.content is None:
|
||||
return CompletionLogProbs()
|
||||
|
||||
tokens: list[str] = []
|
||||
token_logprobs: list[float | None] = []
|
||||
top_logprobs_list: list[dict[str, float] | None] = []
|
||||
text_offset: list[int] = []
|
||||
|
||||
offset = 0
|
||||
for entry in logprobs.content:
|
||||
text_offset.append(offset)
|
||||
tokens.append(entry.token)
|
||||
token_logprobs.append(entry.logprob)
|
||||
top_logprobs_list.append(
|
||||
{t.token: t.logprob for t in entry.top_logprobs}
|
||||
if entry.top_logprobs
|
||||
else None
|
||||
)
|
||||
offset += len(entry.token)
|
||||
|
||||
return CompletionLogProbs(
|
||||
text_offset=text_offset,
|
||||
token_logprobs=token_logprobs,
|
||||
tokens=tokens,
|
||||
top_logprobs=top_logprobs_list,
|
||||
)
|
||||
|
||||
|
||||
class OpenAIServingRender:
|
||||
class ServingRender(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
renderer: BaseRenderer,
|
||||
model_registry: OpenAIModelRegistry,
|
||||
models: OpenAIServingModels | OpenAIModelRegistry,
|
||||
online_renderer: "OnlineRenderer",
|
||||
online_derenderer: "OnlineDerenderer",
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
trust_request_chat_template: bool = False,
|
||||
enable_auto_tools: bool = False,
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
reasoning_parser: str | None = None,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
log_error_stack: bool = False,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
self.model_config = model_config
|
||||
self.renderer = renderer
|
||||
self.model_registry = model_registry
|
||||
self.request_logger = request_logger
|
||||
self.chat_template = chat_template
|
||||
self.chat_template_content_format: ChatTemplateContentFormatOption = (
|
||||
chat_template_content_format
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=online_renderer.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
self.trust_request_chat_template = trust_request_chat_template
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
|
||||
self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
|
||||
self.parser: type[Parser] | None = ParserManager.get_parser(
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
model_name=model_config.model,
|
||||
is_harmony=self.use_harmony,
|
||||
)
|
||||
self.default_chat_template_kwargs: dict[str, Any] = (
|
||||
default_chat_template_kwargs or {}
|
||||
)
|
||||
self.log_error_stack = log_error_stack
|
||||
self.supports_browsing = False
|
||||
self.supports_code_interpreter = False
|
||||
|
||||
self.default_sampling_params = model_config.get_diff_sampling_param()
|
||||
mc = model_config
|
||||
self.online_renderer = online_renderer
|
||||
self.online_derenderer = online_derenderer
|
||||
|
||||
self.default_sampling_params = (
|
||||
online_renderer.model_config.get_diff_sampling_param()
|
||||
)
|
||||
mc = online_renderer.model_config
|
||||
self.override_max_tokens = (
|
||||
self.default_sampling_params.get("max_tokens")
|
||||
if mc.generation_config not in ("auto", "vllm")
|
||||
@@ -273,7 +95,7 @@ class OpenAIServingRender:
|
||||
"Beam search is not supported by the render endpoint"
|
||||
)
|
||||
|
||||
result = await self.render_chat(request, skip_mm_cache=True)
|
||||
result = await self.online_renderer.render_chat(request, skip_mm_cache=True)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
@@ -319,90 +141,6 @@ class OpenAIServingRender:
|
||||
priority=request.priority,
|
||||
)
|
||||
|
||||
async def render_chat(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
|
||||
"""Core preprocessing logic for chat requests (no model/engine check).
|
||||
|
||||
Called directly by render_chat_request and delegated to by
|
||||
OpenAIServingChat.render_chat_request after its engine-aware checks.
|
||||
"""
|
||||
tokenizer = self.renderer.tokenizer
|
||||
|
||||
tool_parser = self.parser.tool_parser_cls if self.parser is not None else None
|
||||
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
# because of issues with pydantic we need to potentially
|
||||
# re-serialize the tool_calls field of the request
|
||||
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
|
||||
_mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
|
||||
_mt.validate_request_params(request)
|
||||
|
||||
# Check if tool parsing is unavailable (common condition)
|
||||
tool_parsing_unavailable = (
|
||||
tool_parser is None
|
||||
and not is_mistral_tokenizer(tokenizer)
|
||||
and not self.use_harmony
|
||||
)
|
||||
|
||||
# Validate tool_choice when tool parsing is required but unavailable
|
||||
if tool_parsing_unavailable and request.tool_choice not in (
|
||||
None,
|
||||
"none",
|
||||
):
|
||||
if request.tool_choice == "auto" and not self.enable_auto_tools:
|
||||
# for hf tokenizers, "auto" tools requires
|
||||
# --enable-auto-tool-choice and --tool-call-parser
|
||||
return self.create_error_response(
|
||||
'"auto" tool choice requires '
|
||||
"--enable-auto-tool-choice and --tool-call-parser to be set"
|
||||
)
|
||||
elif request.tool_choice != "auto":
|
||||
# "required" or named tool requires tool parser
|
||||
return self.create_error_response(
|
||||
f'tool_choice="{request.tool_choice}" requires '
|
||||
"--tool-call-parser to be set"
|
||||
)
|
||||
|
||||
if request.tools is None or (
|
||||
request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
|
||||
):
|
||||
tool_dicts = None
|
||||
else:
|
||||
tool_dicts = [tool.model_dump() for tool in request.tools]
|
||||
|
||||
if not self.use_harmony:
|
||||
# Common case.
|
||||
error_check_ret = self.validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
conversation, engine_inputs = await self.preprocess_chat(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=self.chat_template,
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=self.default_chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
parser=self.parser,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
else:
|
||||
# For GPT-OSS.
|
||||
should_include_tools = tool_dicts is not None
|
||||
conversation, engine_inputs = self._make_request_with_harmony(
|
||||
request, should_include_tools
|
||||
)
|
||||
|
||||
return conversation, engine_inputs
|
||||
|
||||
async def render_completion_request(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
@@ -415,7 +153,9 @@ class OpenAIServingRender:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
result = await self.render_completion(request, skip_mm_cache=True)
|
||||
result = await self.online_renderer.render_completion(
|
||||
request, skip_mm_cache=True
|
||||
)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
generate_requests: list[GenerateRequest] = []
|
||||
@@ -459,37 +199,116 @@ class OpenAIServingRender:
|
||||
|
||||
return generate_requests
|
||||
|
||||
async def render_completion(
|
||||
async def derender_chat_response(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput] | ErrorResponse:
|
||||
"""Core preprocessing logic for completion requests (no model/engine check).
|
||||
request: DerenderChatRequest,
|
||||
) -> ChatCompletionResponse | ErrorResponse:
|
||||
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
|
||||
|
||||
Called directly by render_completion_request and delegated to by
|
||||
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
|
||||
Non-streaming only: expects the complete GenerateResponse with all
|
||||
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
|
||||
|
||||
When ``request.chat_request`` is provided, the parser splits the
|
||||
output into (reasoning, content, tool_calls). Otherwise falls
|
||||
back to plain detokenization.
|
||||
"""
|
||||
# Return error for unsupported features.
|
||||
if request.suffix is not None:
|
||||
return self.create_error_response("suffix is not currently supported")
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
if request.echo and request.prompt_embeds is not None:
|
||||
return self.create_error_response("Echo is unsupported with prompt embeds.")
|
||||
|
||||
if request.prompt_logprobs is not None and request.prompt_embeds is not None:
|
||||
return self.create_error_response(
|
||||
"prompt_logprobs is not compatible with prompt embeds."
|
||||
try:
|
||||
choices = await self.online_derenderer.derender_chat(
|
||||
request.generate_response, request.chat_request
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
|
||||
engine_inputs = await self.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.prompt,
|
||||
prompt_embeds=request.prompt_embeds,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
prompt_tokens = (
|
||||
request.prompt_tokens if request.prompt_tokens is not None else 0
|
||||
)
|
||||
gen = request.generate_response
|
||||
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
return engine_inputs
|
||||
logger.debug(
|
||||
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
|
||||
gen.request_id,
|
||||
request.model,
|
||||
len(choices),
|
||||
completion_tokens,
|
||||
)
|
||||
return ChatCompletionResponse(
|
||||
id=gen.request_id,
|
||||
model=request.model,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=gen.prompt_logprobs,
|
||||
kv_transfer_params=gen.kv_transfer_params,
|
||||
)
|
||||
|
||||
async def derender_completion_response(
|
||||
self,
|
||||
request: DerenderCompletionRequest,
|
||||
) -> CompletionResponse | ErrorResponse:
|
||||
"""Postprocess a list of GenerateResponses into a CompletionResponse.
|
||||
|
||||
Non-streaming only. Mirrors the multi-prompt completions case: one
|
||||
GenerateResponse per prompt, parallel to the list[GenerateRequest]
|
||||
from /v1/completions/render.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
(
|
||||
choices,
|
||||
total_prompt_tokens,
|
||||
total_completion_tokens,
|
||||
) = await self.online_derenderer.derender_completion(
|
||||
request.generate_responses, request.prompt_tokens
|
||||
)
|
||||
|
||||
if not request.generate_responses:
|
||||
return self.create_error_response("generate_responses must not be empty")
|
||||
|
||||
first = request.generate_responses[0]
|
||||
kv_params = first.kv_transfer_params
|
||||
if any(
|
||||
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
|
||||
):
|
||||
logger.warning(
|
||||
"derender_completion: kv_transfer_params differ across responses; "
|
||||
"setting to None on the aggregated response"
|
||||
)
|
||||
kv_params = None
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=total_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_completion request_id=%s model=%s choices=%d"
|
||||
" completion_tokens=%d",
|
||||
first.request_id,
|
||||
request.model,
|
||||
len(choices),
|
||||
total_completion_tokens,
|
||||
)
|
||||
return CompletionResponse(
|
||||
id=first.request_id,
|
||||
model=request.model,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
kv_transfer_params=kv_params,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_mm_features(
|
||||
@@ -529,440 +348,3 @@ class OpenAIServingRender:
|
||||
mm_placeholders=mm_placeholders,
|
||||
kwargs_data=kwargs_data,
|
||||
)
|
||||
|
||||
def _make_request_with_harmony(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
should_include_tools: bool = True,
|
||||
):
|
||||
"""Build Harmony (GPT-OSS) messages and engine prompt from a chat request."""
|
||||
messages: list[OpenAIMessage] = []
|
||||
|
||||
# because of issues with pydantic we need to potentially
|
||||
# re-serialize the tool_calls field of the request
|
||||
# for more info: see comment in `maybe_serialize_tool_calls`
|
||||
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
|
||||
|
||||
chat_messages = list(request.messages)
|
||||
instructions, chat_messages = extract_instructions_from_messages(chat_messages)
|
||||
|
||||
# Add system message.
|
||||
# NOTE: In Chat Completion API, browsing is enabled by default
|
||||
# if the model supports it. TODO: Support browsing.
|
||||
assert not self.supports_browsing
|
||||
assert not self.supports_code_interpreter
|
||||
if (reasoning_effort := request.reasoning_effort) == "none":
|
||||
raise ValueError(f"Harmony does not support {reasoning_effort=}")
|
||||
tools = request.tools if should_include_tools else None
|
||||
messages.extend(
|
||||
build_harmony_preamble(
|
||||
instructions=instructions,
|
||||
tools=tools, # type: ignore[arg-type]
|
||||
reasoning_effort=reasoning_effort,
|
||||
with_custom_tools=should_include_tools,
|
||||
)
|
||||
)
|
||||
|
||||
# Add remaining conversation messages.
|
||||
messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages))
|
||||
|
||||
# Render prompt token ids.
|
||||
prompt_token_ids = render_for_completion(messages)
|
||||
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
|
||||
|
||||
return messages, [engine_input]
|
||||
|
||||
async def derender_chat_response(
|
||||
self,
|
||||
request: DerenderChatRequest,
|
||||
) -> ChatCompletionResponse | ErrorResponse:
|
||||
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
|
||||
|
||||
Non-streaming only: expects the complete GenerateResponse with all
|
||||
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
|
||||
|
||||
When ``request.chat_request`` is provided, the parser splits the
|
||||
output into (reasoning, content, tool_calls). Otherwise falls
|
||||
back to plain detokenization.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
gen = request.generate_response
|
||||
chat_request = request.chat_request
|
||||
choices: list[ChatCompletionResponseChoice] = []
|
||||
|
||||
try:
|
||||
for choice in gen.choices:
|
||||
if not choice.token_ids:
|
||||
raise ValueError(
|
||||
f"choice {choice.index} has empty or null token_ids"
|
||||
)
|
||||
|
||||
resolved_logprobs = (
|
||||
_resolve_logprobs(choice.logprobs, tokenizer)
|
||||
if choice.logprobs is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if self.parser is not None and chat_request is not None:
|
||||
# Parser path: decode with special tokens preserved
|
||||
# so the parser can see markers like </think>,
|
||||
# <tool_call>, or Harmony channel tokens.
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=False
|
||||
)
|
||||
|
||||
chat_template_kwargs: dict[str, Any] = {}
|
||||
if not self.use_harmony:
|
||||
chat_template_kwargs = (
|
||||
chat_request.build_chat_params(
|
||||
self.chat_template,
|
||||
self.chat_template_content_format,
|
||||
)
|
||||
.with_defaults(self.default_chat_template_kwargs)
|
||||
.chat_template_kwargs
|
||||
)
|
||||
|
||||
parser = self.parser(
|
||||
tokenizer,
|
||||
chat_request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
decoded_text,
|
||||
chat_request,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
model_output_token_ids=choice.token_ids,
|
||||
)
|
||||
|
||||
if not getattr(chat_request, "include_reasoning", True):
|
||||
reasoning = None
|
||||
|
||||
tc_items = (
|
||||
[
|
||||
ToolCall(
|
||||
id=random_uuid(),
|
||||
function=tc,
|
||||
)
|
||||
for tc in tool_calls
|
||||
]
|
||||
if tool_calls
|
||||
else []
|
||||
)
|
||||
|
||||
message = ChatMessage(
|
||||
role="assistant",
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tc_items,
|
||||
)
|
||||
else:
|
||||
# No parser: plain detokenization.
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=True
|
||||
)
|
||||
message = ChatMessage(role="assistant", content=decoded_text)
|
||||
|
||||
choices.append(
|
||||
ChatCompletionResponseChoice(
|
||||
index=choice.index,
|
||||
message=message,
|
||||
logprobs=resolved_logprobs,
|
||||
finish_reason=choice.finish_reason,
|
||||
)
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
|
||||
prompt_tokens = (
|
||||
request.prompt_tokens if request.prompt_tokens is not None else 0
|
||||
)
|
||||
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
|
||||
gen.request_id,
|
||||
request.model,
|
||||
len(choices),
|
||||
completion_tokens,
|
||||
)
|
||||
return ChatCompletionResponse(
|
||||
id=gen.request_id,
|
||||
model=request.model,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
prompt_logprobs=gen.prompt_logprobs,
|
||||
kv_transfer_params=gen.kv_transfer_params,
|
||||
)
|
||||
|
||||
async def derender_completion_response(
|
||||
self,
|
||||
request: DerenderCompletionRequest,
|
||||
) -> CompletionResponse | ErrorResponse:
|
||||
"""Postprocess a list of GenerateResponses into a CompletionResponse.
|
||||
|
||||
Non-streaming only. Mirrors the multi-prompt completions case: one
|
||||
GenerateResponse per prompt, parallel to the list[GenerateRequest]
|
||||
from /v1/completions/render.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
n = len(request.generate_responses)
|
||||
prompt_tokens_list: list[int] = (
|
||||
request.prompt_tokens if request.prompt_tokens is not None else [0] * n
|
||||
)
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
choices: list[CompletionResponseChoice] = []
|
||||
total_prompt_tokens = 0
|
||||
total_completion_tokens = 0
|
||||
index = 0
|
||||
|
||||
for gen, pt in zip(request.generate_responses, prompt_tokens_list):
|
||||
for choice in gen.choices:
|
||||
if not choice.token_ids:
|
||||
return self.create_error_response(
|
||||
f"choice {choice.index} in response {gen.request_id} "
|
||||
"has empty or null token_ids"
|
||||
)
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=True
|
||||
)
|
||||
completion_logprobs = None
|
||||
if choice.logprobs is not None:
|
||||
resolved = _resolve_logprobs(choice.logprobs, tokenizer)
|
||||
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
|
||||
resolved
|
||||
)
|
||||
choices.append(
|
||||
CompletionResponseChoice(
|
||||
index=index,
|
||||
text=decoded_text,
|
||||
finish_reason=choice.finish_reason,
|
||||
logprobs=completion_logprobs,
|
||||
)
|
||||
)
|
||||
total_completion_tokens += len(choice.token_ids)
|
||||
index += 1
|
||||
total_prompt_tokens += pt
|
||||
|
||||
if not request.generate_responses:
|
||||
return self.create_error_response("generate_responses must not be empty")
|
||||
|
||||
first = request.generate_responses[0]
|
||||
kv_params = first.kv_transfer_params
|
||||
if any(
|
||||
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
|
||||
):
|
||||
logger.warning(
|
||||
"derender_completion: kv_transfer_params differ across responses; "
|
||||
"setting to None on the aggregated response"
|
||||
)
|
||||
kv_params = None
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=total_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"derender_completion request_id=%s model=%s choices=%d"
|
||||
" completion_tokens=%d",
|
||||
first.request_id,
|
||||
request.model,
|
||||
len(choices),
|
||||
total_completion_tokens,
|
||||
)
|
||||
return CompletionResponse(
|
||||
id=first.request_id,
|
||||
model=request.model,
|
||||
created=int(time.time()),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
kv_transfer_params=kv_params,
|
||||
)
|
||||
|
||||
def create_error_response(
|
||||
self,
|
||||
message: str | Exception,
|
||||
err_type: str = "BadRequestError",
|
||||
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
|
||||
param: str | None = None,
|
||||
) -> ErrorResponse:
|
||||
return create_error_response(message, err_type, status_code, param)
|
||||
|
||||
async def _check_model(
|
||||
self,
|
||||
request: Any,
|
||||
) -> ErrorResponse | None:
|
||||
return await self.model_registry.check_model(request.model)
|
||||
|
||||
def validate_chat_template(
|
||||
self,
|
||||
request_chat_template: str | None,
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
trust_request_chat_template: bool,
|
||||
) -> ErrorResponse | None:
|
||||
"""Copied from OpenAIServing._validate_chat_template."""
|
||||
if not trust_request_chat_template and (
|
||||
request_chat_template is not None
|
||||
or (
|
||||
chat_template_kwargs
|
||||
and chat_template_kwargs.get("chat_template") is not None
|
||||
)
|
||||
):
|
||||
return self.create_error_response(
|
||||
"Chat template is passed with request, but "
|
||||
"--trust-request-chat-template is not set. "
|
||||
"Refused request with untrusted chat template."
|
||||
)
|
||||
return None
|
||||
|
||||
async def preprocess_completion(
|
||||
self,
|
||||
request: Any,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
||||
prompt_embeds: bytes | list[bytes] | None,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput]:
|
||||
"""Copied from OpenAIServing._preprocess_completion."""
|
||||
prompts = list[SingletonPrompt | bytes]()
|
||||
if prompt_embeds is not None: # embeds take higher priority
|
||||
prompts.extend(prompt_to_seq(prompt_embeds))
|
||||
if prompt_input is not None:
|
||||
prompts.extend(prompt_to_seq(prompt_input))
|
||||
return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache)
|
||||
|
||||
async def preprocess_cmpl(
|
||||
self,
|
||||
request: Any,
|
||||
prompts: Sequence[PromptType | bytes],
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput]:
|
||||
"""Copied from OpenAIServing._preprocess_cmpl."""
|
||||
renderer = self.renderer
|
||||
model_config = self.model_config
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(model_config, prompt)
|
||||
)
|
||||
for prompt in prompts
|
||||
]
|
||||
tok_params = request.build_tok_params(model_config)
|
||||
|
||||
return await renderer.render_cmpl_async(
|
||||
parsed_prompts,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
async def preprocess_chat(
|
||||
self,
|
||||
request: Any,
|
||||
messages: list[Any],
|
||||
default_template: str | None,
|
||||
default_template_content_format: ChatTemplateContentFormatOption,
|
||||
default_template_kwargs: dict[str, Any] | None,
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
parser: type[Parser] | None = None,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||
"""Copied from OpenAIServing._preprocess_chat."""
|
||||
renderer = self.renderer
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
default_template_kwargs = merge_kwargs(
|
||||
default_template_kwargs,
|
||||
dict(
|
||||
tools=tool_dicts,
|
||||
tokenize=(
|
||||
is_mistral_tokenizer(renderer.tokenizer)
|
||||
or self.model_config.enable_prompt_embeds
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
default_template, default_template_content_format
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||
)
|
||||
|
||||
(conversation,), (engine_input,) = await renderer.render_chat_async(
|
||||
[messages],
|
||||
chat_params,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
# tool parsing is done only if a tool_parser has been set and if
|
||||
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
|
||||
# is set, we want to prevent parsing a tool_call hallucinated by the LLM
|
||||
#
|
||||
# Exception: Mistral grammar-capable tokenizers always call
|
||||
# adjust_request — even for tool_choice="none" — so that the grammar
|
||||
# factory can prevent special-token leakage.
|
||||
if parser is not None:
|
||||
tokenizer = renderer.get_tokenizer()
|
||||
tool_parser = parser.tool_parser_cls
|
||||
tool_choice = getattr(request, "tool_choice", "none")
|
||||
is_mistral_grammar_eligible = (
|
||||
tool_parser is not None
|
||||
and is_mistral_tool_parser(tool_parser)
|
||||
and is_mistral_tokenizer(tokenizer)
|
||||
and tokenizer.supports_grammar
|
||||
)
|
||||
should_adjust_request = (
|
||||
parser.reasoning_parser_cls is not None
|
||||
or tool_choice != "none"
|
||||
or is_mistral_grammar_eligible
|
||||
)
|
||||
if should_adjust_request:
|
||||
if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
|
||||
msg = (
|
||||
"Tool usage is only supported "
|
||||
"for Chat Completions API or Responses API requests, "
|
||||
f"but got {type(request).__name__}"
|
||||
)
|
||||
raise NotImplementedError(msg)
|
||||
request = parser(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
chat_template_kwargs=chat_params.chat_template_kwargs,
|
||||
).adjust_request(
|
||||
request=request,
|
||||
)
|
||||
|
||||
return conversation, [engine_input]
|
||||
|
||||
@@ -12,7 +12,6 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
DetokenizeRequest,
|
||||
DetokenizeResponse,
|
||||
@@ -24,6 +23,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import TokensPrompt, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -33,22 +33,22 @@ class ServingTokenization(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels | OpenAIModelRegistry,
|
||||
openai_serving_render: OpenAIServingRender,
|
||||
online_renderer: OnlineRenderer,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
trust_request_chat_template: bool = False,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=openai_serving_render.model_config,
|
||||
model_config=online_renderer.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.renderer = openai_serving_render.renderer
|
||||
self.openai_serving_render = openai_serving_render
|
||||
self.renderer = online_renderer.renderer
|
||||
self.online_renderer = online_renderer
|
||||
self.chat_template = chat_template
|
||||
self.chat_template_content_format: Final = chat_template_content_format
|
||||
self.default_chat_template_kwargs = default_chat_template_kwargs or {}
|
||||
@@ -73,7 +73,7 @@ class ServingTokenization(BaseServing):
|
||||
if request.tools is None
|
||||
else [tool.model_dump() for tool in request.tools]
|
||||
)
|
||||
error_check_ret = self.openai_serving_render.validate_chat_template(
|
||||
error_check_ret = self.online_renderer.validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
@@ -81,7 +81,7 @@ class ServingTokenization(BaseServing):
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
|
||||
_, engine_inputs = await self.online_renderer.preprocess_chat(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=self.chat_template,
|
||||
@@ -91,7 +91,7 @@ class ServingTokenization(BaseServing):
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
else:
|
||||
engine_inputs = await self.openai_serving_render.preprocess_completion(
|
||||
engine_inputs = await self.online_renderer.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.prompt,
|
||||
prompt_embeds=None,
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProbs,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponseChoice,
|
||||
ChatMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionLogProbs,
|
||||
CompletionResponseChoice,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import ToolCall
|
||||
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
|
||||
from vllm.entrypoints.serve.disagg.protocol import GenerateResponse
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OnlineDerenderer:
|
||||
def __init__(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
renderer: BaseRenderer,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
trust_request_chat_template: bool = False,
|
||||
enable_auto_tools: bool = False,
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
reasoning_parser: str | None = None,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
log_error_stack: bool = False,
|
||||
) -> None:
|
||||
self.model_config = model_config
|
||||
self.renderer = renderer
|
||||
self.request_logger = request_logger
|
||||
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
|
||||
self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
|
||||
self.parser: type[Parser] | None = ParserManager.get_parser(
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
model_name=model_config.model,
|
||||
is_harmony=self.use_harmony,
|
||||
)
|
||||
|
||||
self.chat_template = chat_template
|
||||
self.chat_template_content_format: ChatTemplateContentFormatOption = (
|
||||
chat_template_content_format
|
||||
)
|
||||
self.default_chat_template_kwargs: dict[str, Any] = (
|
||||
default_chat_template_kwargs or {}
|
||||
)
|
||||
self.trust_request_chat_template = trust_request_chat_template
|
||||
|
||||
self.log_error_stack = log_error_stack
|
||||
self.supports_browsing = False
|
||||
self.supports_code_interpreter = False
|
||||
|
||||
async def derender_chat(
|
||||
self,
|
||||
generate_response: GenerateResponse,
|
||||
chat_request: ChatCompletionRequest | None = None,
|
||||
) -> list[ChatCompletionResponseChoice]:
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
choices: list[ChatCompletionResponseChoice] = []
|
||||
|
||||
for choice in generate_response.choices:
|
||||
if not choice.token_ids:
|
||||
raise ValueError(f"choice {choice.index} has empty or null token_ids")
|
||||
|
||||
resolved_logprobs = (
|
||||
_resolve_logprobs(choice.logprobs, tokenizer)
|
||||
if choice.logprobs is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if self.parser is not None and chat_request is not None:
|
||||
# Parser path: decode with special tokens preserved
|
||||
# so the parser can see markers like </think>,
|
||||
# <tool_call>, or Harmony channel tokens.
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=False
|
||||
)
|
||||
|
||||
chat_template_kwargs: dict[str, Any] = {}
|
||||
if not self.use_harmony:
|
||||
chat_template_kwargs = (
|
||||
chat_request.build_chat_params(
|
||||
self.chat_template,
|
||||
self.chat_template_content_format,
|
||||
)
|
||||
.with_defaults(self.default_chat_template_kwargs)
|
||||
.chat_template_kwargs
|
||||
)
|
||||
|
||||
parser = self.parser(
|
||||
tokenizer,
|
||||
chat_request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
decoded_text,
|
||||
chat_request,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
model_output_token_ids=choice.token_ids,
|
||||
)
|
||||
|
||||
if not getattr(chat_request, "include_reasoning", True):
|
||||
reasoning = None
|
||||
|
||||
tc_items = (
|
||||
[
|
||||
ToolCall(
|
||||
id=random_uuid(),
|
||||
function=tc,
|
||||
)
|
||||
for tc in tool_calls
|
||||
]
|
||||
if tool_calls
|
||||
else []
|
||||
)
|
||||
|
||||
message = ChatMessage(
|
||||
role="assistant",
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tc_items,
|
||||
)
|
||||
else:
|
||||
# No parser: plain detokenization.
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=True
|
||||
)
|
||||
message = ChatMessage(role="assistant", content=decoded_text)
|
||||
|
||||
choices.append(
|
||||
ChatCompletionResponseChoice(
|
||||
index=choice.index,
|
||||
message=message,
|
||||
logprobs=resolved_logprobs,
|
||||
finish_reason=choice.finish_reason,
|
||||
)
|
||||
)
|
||||
|
||||
return choices
|
||||
|
||||
async def derender_completion(
|
||||
self,
|
||||
generate_responses: list[GenerateResponse],
|
||||
prompt_tokens: list[int] | None = None,
|
||||
) -> tuple[list[CompletionResponseChoice], int, int]:
|
||||
n = len(generate_responses)
|
||||
prompt_tokens_list: list[int] = (
|
||||
prompt_tokens if prompt_tokens is not None else [0] * n
|
||||
)
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
choices: list[CompletionResponseChoice] = []
|
||||
total_prompt_tokens = 0
|
||||
total_completion_tokens = 0
|
||||
index = 0
|
||||
|
||||
for gen, pt in zip(generate_responses, prompt_tokens_list):
|
||||
for choice in gen.choices:
|
||||
if not choice.token_ids:
|
||||
raise ValueError(
|
||||
f"choice {choice.index} in response {gen.request_id} "
|
||||
"has empty or null token_ids"
|
||||
)
|
||||
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=True
|
||||
)
|
||||
completion_logprobs = None
|
||||
if choice.logprobs is not None:
|
||||
resolved = _resolve_logprobs(choice.logprobs, tokenizer)
|
||||
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
|
||||
resolved
|
||||
)
|
||||
choices.append(
|
||||
CompletionResponseChoice(
|
||||
index=index,
|
||||
text=decoded_text,
|
||||
finish_reason=choice.finish_reason,
|
||||
logprobs=completion_logprobs,
|
||||
)
|
||||
)
|
||||
total_completion_tokens += len(choice.token_ids)
|
||||
index += 1
|
||||
total_prompt_tokens += pt
|
||||
|
||||
return choices, total_prompt_tokens, total_completion_tokens
|
||||
|
||||
|
||||
def _parse_token_id_placeholder(token: str) -> int | None:
|
||||
"""Extract token ID from a 'token_id:N' placeholder string."""
|
||||
if not token.startswith("token_id:"):
|
||||
return None
|
||||
try:
|
||||
return int(token[len("token_id:") :])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _correct_decoded_token(
|
||||
token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike
|
||||
) -> str:
|
||||
"""Use preceding tokens as context to fix U+FFFD from byte-fallback.
|
||||
|
||||
Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py.
|
||||
"""
|
||||
max_ctx = min(len(context_token_ids), 4)
|
||||
|
||||
for num_ctx in range(1, max_ctx + 1):
|
||||
context = context_token_ids[-num_ctx:]
|
||||
full_decoded = tokenizer.decode(context + [token_id])
|
||||
|
||||
if full_decoded.endswith("�"):
|
||||
continue
|
||||
|
||||
clean_end = len(context)
|
||||
for j in range(len(context) - 1, -1, -1):
|
||||
if tokenizer.decode([context[j]]).endswith("�"):
|
||||
clean_end = j
|
||||
else:
|
||||
break
|
||||
|
||||
clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else ""
|
||||
|
||||
if full_decoded.startswith(clean_prefix):
|
||||
return full_decoded[len(clean_prefix) :]
|
||||
|
||||
common_len = 0
|
||||
for a, b in zip(clean_prefix, full_decoded):
|
||||
if a != b:
|
||||
break
|
||||
common_len += 1
|
||||
return full_decoded[common_len:]
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def _resolve_logprobs(
|
||||
logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Resolve token_id:N placeholders in a ChatCompletionLogProbs object."""
|
||||
if logprobs.content is None:
|
||||
return logprobs
|
||||
|
||||
context_token_ids: list[int] = []
|
||||
resolved_content = []
|
||||
|
||||
for entry in logprobs.content:
|
||||
token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
|
||||
sampled_id = _parse_token_id_placeholder(entry.token)
|
||||
|
||||
if token_str.endswith("�") and sampled_id is not None:
|
||||
token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer)
|
||||
token_bytes = list(token_str.encode("utf-8"))
|
||||
|
||||
resolved_top = []
|
||||
for top in entry.top_logprobs:
|
||||
top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
|
||||
top_id = _parse_token_id_placeholder(top.token)
|
||||
if top_str.endswith("�") and top_id is not None:
|
||||
top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer)
|
||||
top_bytes = list(top_str.encode("utf-8"))
|
||||
resolved_top.append(
|
||||
top.model_copy(update={"token": top_str, "bytes": top_bytes})
|
||||
)
|
||||
|
||||
resolved_content.append(
|
||||
entry.model_copy(
|
||||
update={
|
||||
"token": token_str,
|
||||
"bytes": token_bytes,
|
||||
"top_logprobs": resolved_top,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
if sampled_id is not None:
|
||||
context_token_ids.append(sampled_id)
|
||||
|
||||
return ChatCompletionLogProbs(content=resolved_content)
|
||||
|
||||
|
||||
def _convert_chat_logprobs_to_completion_logprobs(
|
||||
logprobs: ChatCompletionLogProbs,
|
||||
) -> CompletionLogProbs:
|
||||
"""Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
|
||||
(parallel flat lists) as required by the /v1/completions response schema."""
|
||||
if logprobs.content is None:
|
||||
return CompletionLogProbs()
|
||||
|
||||
tokens: list[str] = []
|
||||
token_logprobs: list[float | None] = []
|
||||
top_logprobs_list: list[dict[str, float] | None] = []
|
||||
text_offset: list[int] = []
|
||||
|
||||
offset = 0
|
||||
for entry in logprobs.content:
|
||||
text_offset.append(offset)
|
||||
tokens.append(entry.token)
|
||||
token_logprobs.append(entry.logprob)
|
||||
top_logprobs_list.append(
|
||||
{t.token: t.logprob for t in entry.top_logprobs}
|
||||
if entry.top_logprobs
|
||||
else None
|
||||
)
|
||||
offset += len(entry.token)
|
||||
|
||||
return CompletionLogProbs(
|
||||
text_offset=text_offset,
|
||||
token_logprobs=token_logprobs,
|
||||
tokens=tokens,
|
||||
top_logprobs=top_logprobs_list,
|
||||
)
|
||||
@@ -0,0 +1,417 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
from openai_harmony import Message as OpenAIMessage
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
build_harmony_preamble,
|
||||
extract_instructions_from_messages,
|
||||
parse_chat_inputs_to_harmony_messages,
|
||||
render_for_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.utils.error_response import create_error_response
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import (
|
||||
EngineInput,
|
||||
PromptType,
|
||||
SingletonPrompt,
|
||||
tokens_input,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
|
||||
from vllm.utils.mistral import mt as _mt
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OnlineRenderer:
|
||||
def __init__(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
renderer: BaseRenderer,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
trust_request_chat_template: bool = False,
|
||||
enable_auto_tools: bool = False,
|
||||
exclude_tools_when_tool_choice_none: bool = False,
|
||||
tool_parser: str | None = None,
|
||||
reasoning_parser: str | None = None,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
log_error_stack: bool = False,
|
||||
) -> None:
|
||||
self.model_config = model_config
|
||||
self.renderer = renderer
|
||||
self.request_logger = request_logger
|
||||
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
|
||||
self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
|
||||
self.parser: type[Parser] | None = ParserManager.get_parser(
|
||||
tool_parser_name=tool_parser,
|
||||
reasoning_parser_name=reasoning_parser,
|
||||
enable_auto_tools=enable_auto_tools,
|
||||
model_name=model_config.model,
|
||||
is_harmony=self.use_harmony,
|
||||
)
|
||||
|
||||
self.chat_template = chat_template
|
||||
self.chat_template_content_format: ChatTemplateContentFormatOption = (
|
||||
chat_template_content_format
|
||||
)
|
||||
self.default_chat_template_kwargs: dict[str, Any] = (
|
||||
default_chat_template_kwargs or {}
|
||||
)
|
||||
self.trust_request_chat_template = trust_request_chat_template
|
||||
|
||||
self.log_error_stack = log_error_stack
|
||||
self.supports_browsing = False
|
||||
self.supports_code_interpreter = False
|
||||
|
||||
async def render_chat(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
|
||||
"""Core preprocessing logic for chat requests (no model/engine check).
|
||||
|
||||
Called directly by render_chat_request and delegated to by
|
||||
OpenAIServingChat.render_chat_request after its engine-aware checks.
|
||||
"""
|
||||
tokenizer = self.renderer.tokenizer
|
||||
|
||||
tool_parser = self.parser.tool_parser_cls if self.parser is not None else None
|
||||
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
# because of issues with pydantic we need to potentially
|
||||
# re-serialize the tool_calls field of the request
|
||||
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
|
||||
_mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
|
||||
_mt.validate_request_params(request)
|
||||
|
||||
# Check if tool parsing is unavailable (common condition)
|
||||
tool_parsing_unavailable = (
|
||||
tool_parser is None
|
||||
and not is_mistral_tokenizer(tokenizer)
|
||||
and not self.use_harmony
|
||||
)
|
||||
|
||||
# Validate tool_choice when tool parsing is required but unavailable
|
||||
if tool_parsing_unavailable and request.tool_choice not in (
|
||||
None,
|
||||
"none",
|
||||
):
|
||||
if request.tool_choice == "auto" and not self.enable_auto_tools:
|
||||
# for hf tokenizers, "auto" tools requires
|
||||
# --enable-auto-tool-choice and --tool-call-parser
|
||||
return self.create_error_response(
|
||||
'"auto" tool choice requires '
|
||||
"--enable-auto-tool-choice and --tool-call-parser to be set"
|
||||
)
|
||||
elif request.tool_choice != "auto":
|
||||
# "required" or named tool requires tool parser
|
||||
return self.create_error_response(
|
||||
f'tool_choice="{request.tool_choice}" requires '
|
||||
"--tool-call-parser to be set"
|
||||
)
|
||||
|
||||
if request.tools is None or (
|
||||
request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
|
||||
):
|
||||
tool_dicts = None
|
||||
else:
|
||||
tool_dicts = [tool.model_dump() for tool in request.tools]
|
||||
|
||||
if not self.use_harmony:
|
||||
# Common case.
|
||||
error_check_ret = self.validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
conversation, engine_inputs = await self.preprocess_chat(
|
||||
request,
|
||||
request.messages,
|
||||
default_template=self.chat_template,
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=self.default_chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
parser=self.parser,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
else:
|
||||
# For GPT-OSS.
|
||||
should_include_tools = tool_dicts is not None
|
||||
conversation, engine_inputs = self._make_request_with_harmony(
|
||||
request, should_include_tools
|
||||
)
|
||||
|
||||
return conversation, engine_inputs
|
||||
|
||||
def _make_request_with_harmony(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
should_include_tools: bool = True,
|
||||
):
|
||||
"""Build Harmony (GPT-OSS) messages and engine prompt from a chat request."""
|
||||
messages: list[OpenAIMessage] = []
|
||||
|
||||
# because of issues with pydantic we need to potentially
|
||||
# re-serialize the tool_calls field of the request
|
||||
# for more info: see comment in `maybe_serialize_tool_calls`
|
||||
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
|
||||
|
||||
chat_messages = list(request.messages)
|
||||
instructions, chat_messages = extract_instructions_from_messages(chat_messages)
|
||||
|
||||
# Add system message.
|
||||
# NOTE: In Chat Completion API, browsing is enabled by default
|
||||
# if the model supports it. TODO: Support browsing.
|
||||
assert not self.supports_browsing
|
||||
assert not self.supports_code_interpreter
|
||||
if (reasoning_effort := request.reasoning_effort) == "none":
|
||||
raise ValueError(f"Harmony does not support {reasoning_effort=}")
|
||||
tools = request.tools if should_include_tools else None
|
||||
messages.extend(
|
||||
build_harmony_preamble(
|
||||
instructions=instructions,
|
||||
tools=tools, # type: ignore[arg-type]
|
||||
reasoning_effort=reasoning_effort,
|
||||
with_custom_tools=should_include_tools,
|
||||
)
|
||||
)
|
||||
|
||||
# Add remaining conversation messages.
|
||||
messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages))
|
||||
|
||||
# Render prompt token ids.
|
||||
prompt_token_ids = render_for_completion(messages)
|
||||
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
|
||||
|
||||
return messages, [engine_input]
|
||||
|
||||
async def render_completion(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput] | ErrorResponse:
|
||||
"""Core preprocessing logic for completion requests (no model/engine check).
|
||||
|
||||
Called directly by render_completion_request and delegated to by
|
||||
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
|
||||
"""
|
||||
# Return error for unsupported features.
|
||||
if request.suffix is not None:
|
||||
return self.create_error_response("suffix is not currently supported")
|
||||
|
||||
if request.echo and request.prompt_embeds is not None:
|
||||
return self.create_error_response("Echo is unsupported with prompt embeds.")
|
||||
|
||||
if request.prompt_logprobs is not None and request.prompt_embeds is not None:
|
||||
return self.create_error_response(
|
||||
"prompt_logprobs is not compatible with prompt embeds."
|
||||
)
|
||||
|
||||
engine_inputs = await self.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.prompt,
|
||||
prompt_embeds=request.prompt_embeds,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
return engine_inputs
|
||||
|
||||
def create_error_response(
|
||||
self,
|
||||
message: str | Exception,
|
||||
err_type: str = "BadRequestError",
|
||||
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
|
||||
param: str | None = None,
|
||||
) -> ErrorResponse:
|
||||
return create_error_response(message, err_type, status_code, param)
|
||||
|
||||
def validate_chat_template(
|
||||
self,
|
||||
request_chat_template: str | None,
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
trust_request_chat_template: bool,
|
||||
) -> ErrorResponse | None:
|
||||
"""Copied from OpenAIServing._validate_chat_template."""
|
||||
if not trust_request_chat_template and (
|
||||
request_chat_template is not None
|
||||
or (
|
||||
chat_template_kwargs
|
||||
and chat_template_kwargs.get("chat_template") is not None
|
||||
)
|
||||
):
|
||||
return self.create_error_response(
|
||||
"Chat template is passed with request, but "
|
||||
"--trust-request-chat-template is not set. "
|
||||
"Refused request with untrusted chat template."
|
||||
)
|
||||
return None
|
||||
|
||||
async def preprocess_completion(
|
||||
self,
|
||||
request: Any,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
||||
prompt_embeds: bytes | list[bytes] | None,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput]:
|
||||
"""Copied from OpenAIServing._preprocess_completion."""
|
||||
prompts = list[SingletonPrompt | bytes]()
|
||||
if prompt_embeds is not None: # embeds take higher priority
|
||||
prompts.extend(prompt_to_seq(prompt_embeds))
|
||||
if prompt_input is not None:
|
||||
prompts.extend(prompt_to_seq(prompt_input))
|
||||
return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache)
|
||||
|
||||
async def preprocess_cmpl(
|
||||
self,
|
||||
request: Any,
|
||||
prompts: Sequence[PromptType | bytes],
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> list[EngineInput]:
|
||||
"""Copied from OpenAIServing._preprocess_cmpl."""
|
||||
renderer = self.renderer
|
||||
model_config = self.model_config
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(model_config, prompt)
|
||||
)
|
||||
for prompt in prompts
|
||||
]
|
||||
tok_params = request.build_tok_params(model_config)
|
||||
|
||||
return await renderer.render_cmpl_async(
|
||||
parsed_prompts,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
async def preprocess_chat(
|
||||
self,
|
||||
request: Any,
|
||||
messages: list[Any],
|
||||
default_template: str | None,
|
||||
default_template_content_format: ChatTemplateContentFormatOption,
|
||||
default_template_kwargs: dict[str, Any] | None,
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
parser: type[Parser] | None = None,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> tuple[list[ConversationMessage], list[EngineInput]]:
|
||||
"""Copied from OpenAIServing._preprocess_chat."""
|
||||
renderer = self.renderer
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
default_template_kwargs = merge_kwargs(
|
||||
default_template_kwargs,
|
||||
dict(
|
||||
tools=tool_dicts,
|
||||
tokenize=(
|
||||
is_mistral_tokenizer(renderer.tokenizer)
|
||||
or self.model_config.enable_prompt_embeds
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
default_template, default_template_content_format
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||
)
|
||||
|
||||
(conversation,), (engine_input,) = await renderer.render_chat_async(
|
||||
[messages],
|
||||
chat_params,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
# tool parsing is done only if a tool_parser has been set and if
|
||||
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
|
||||
# is set, we want to prevent parsing a tool_call hallucinated by the LLM
|
||||
#
|
||||
# Exception: Mistral grammar-capable tokenizers always call
|
||||
# adjust_request — even for tool_choice="none" — so that the grammar
|
||||
# factory can prevent special-token leakage.
|
||||
if parser is not None:
|
||||
tokenizer = renderer.get_tokenizer()
|
||||
tool_parser = parser.tool_parser_cls
|
||||
tool_choice = getattr(request, "tool_choice", "none")
|
||||
is_mistral_grammar_eligible = (
|
||||
tool_parser is not None
|
||||
and is_mistral_tool_parser(tool_parser)
|
||||
and is_mistral_tokenizer(tokenizer)
|
||||
and tokenizer.supports_grammar
|
||||
)
|
||||
should_adjust_request = (
|
||||
parser.reasoning_parser_cls is not None
|
||||
or tool_choice != "none"
|
||||
or is_mistral_grammar_eligible
|
||||
)
|
||||
if should_adjust_request:
|
||||
if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
|
||||
msg = (
|
||||
"Tool usage is only supported "
|
||||
"for Chat Completions API or Responses API requests, "
|
||||
f"but got {type(request).__name__}"
|
||||
)
|
||||
raise NotImplementedError(msg)
|
||||
request = parser(
|
||||
tokenizer,
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
chat_template_kwargs=chat_params.chat_template_kwargs,
|
||||
).adjust_request(
|
||||
request=request,
|
||||
)
|
||||
|
||||
return conversation, [engine_input]
|
||||
Reference in New Issue
Block a user