diff --git a/tests/entrypoints/openai/chat_completion/test_chat_error.py b/tests/entrypoints/openai/chat_completion/test_chat_error.py index e099c282f4..3eea57d3f5 100644 --- a/tests/entrypoints/openai/chat_completion/test_chat_error.py +++ b/tests/entrypoints/openai/chat_completion/test_chat_error.py @@ -17,9 +17,11 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.render.serving import OpenAIServingRender +from vllm.entrypoints.serve.render.serving import ServingRender from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers.hf import HfRenderer +from vllm.renderers.online_derenderer import OnlineDerenderer +from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.engine.async_llm import AsyncLLM @@ -88,19 +90,19 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat: engine_client=engine, base_model_paths=BASE_MODEL_PATHS, ) - serving_render = OpenAIServingRender( + online_renderer = OnlineRenderer( model_config=engine.model_config, renderer=engine.renderer, - model_registry=models.registry, request_logger=None, chat_template=None, chat_template_content_format="auto", ) + serving_chat = OpenAIServingChat( engine, models, response_role="assistant", - openai_serving_render=serving_render, + online_renderer=online_renderer, request_logger=None, chat_template=None, chat_template_content_format="auto", @@ -113,7 +115,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat: [{"prompt_token_ids": [1, 2, 3]}], ) - serving_chat.openai_serving_render.preprocess_chat = AsyncMock( + serving_chat.online_renderer.preprocess_chat = AsyncMock( side_effect=_fake_preprocess_chat ) return serving_chat @@ -187,13 +189,46 @@ async def test_openai_chat_keeps_mm_cache_for_engine_execution(): assert isinstance(result, tuple) assert ( - serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[ - "skip_mm_cache" - ] + serving_chat.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"] is False ) +def _build_serving_render(engine: AsyncLLM) -> ServingRender: + models = OpenAIServingModels( + engine_client=engine, + base_model_paths=BASE_MODEL_PATHS, + ) + online_renderer = OnlineRenderer( + model_config=engine.model_config, + renderer=engine.renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + online_derenderer = OnlineDerenderer( + model_config=engine.model_config, + renderer=engine.renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + + serving_render = ServingRender(models, online_renderer, online_derenderer) + + async def _fake_preprocess_chat(*args, **kwargs): + # return conversation, engine_inputs + return ( + [{"role": "user", "content": "Test"}], + [{"prompt_token_ids": [1, 2, 3]}], + ) + + serving_render.online_renderer.preprocess_chat = AsyncMock( + side_effect=_fake_preprocess_chat + ) + return serving_render + + @pytest.mark.asyncio async def test_renderer_only_chat_request_skips_mm_cache(): mock_engine = MagicMock(spec=AsyncLLM) @@ -202,20 +237,18 @@ async def test_renderer_only_chat_request_skips_mm_cache(): mock_engine.input_processor = MagicMock() mock_engine.renderer = _build_renderer(mock_engine.model_config) - serving_chat = _build_serving_chat(mock_engine) + serving_render = _build_serving_render(mock_engine) request = ChatCompletionRequest( model=MODEL_NAME, messages=[{"role": "user", "content": "Test prompt"}], ) - result = await serving_chat.openai_serving_render.render_chat_request(request) + result = await serving_render.render_chat_request(request) assert result.token_ids == [1, 2, 3] assert ( - serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[ - "skip_mm_cache" - ] + serving_render.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"] is True ) diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 20fa75a770..3802b7e3e5 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -38,7 +38,6 @@ from vllm.entrypoints.openai.models.serving import ( OpenAIServingModels, ) from vllm.entrypoints.openai.parser.harmony_utils import get_encoding -from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.exceptions import VLLMValidationError from vllm.inputs import TokensPrompt from vllm.multimodal.inputs import PlaceholderRange @@ -46,6 +45,7 @@ from vllm.outputs import CompletionOutput, RequestOutput from vllm.parser import HarmonyParser from vllm.renderers.hf import HfRenderer from vllm.renderers.mistral import MistralRenderer +from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers import get_tokenizer from vllm.tokenizers.mistral import MistralTokenizer from vllm.tokenizers.registry import cached_tokenizer_from_config @@ -567,13 +567,12 @@ def _build_renderer(model_config: MockModelConfig): ) -def _build_serving_render( +def _build_online_renderer( engine, model_registry: OpenAIModelRegistry -) -> OpenAIServingRender: - return OpenAIServingRender( +) -> OnlineRenderer: + return OnlineRenderer( model_config=engine.model_config, renderer=engine.renderer, - model_registry=model_registry, request_logger=None, chat_template=CHAT_TEMPLATE, chat_template_content_format="auto", @@ -591,13 +590,13 @@ def _build_serving_chat( engine_client=engine, base_model_paths=BASE_MODEL_PATHS, ) - openai_serving_render = _build_serving_render(engine, models.registry) + online_renderer = _build_online_renderer(engine, models.registry) serving_chat = OpenAIServingChat( engine, models, response_role="assistant", - openai_serving_render=openai_serving_render, + online_renderer=online_renderer, chat_template=CHAT_TEMPLATE, chat_template_content_format="auto", request_logger=None, @@ -621,13 +620,13 @@ async def _async_serving_chat_init(): engine = MockEngine() models = OpenAIServingModels(engine, BASE_MODEL_PATHS) - openai_serving_render = _build_serving_render(engine, models.registry) + online_renderer = _build_online_renderer(engine, models.registry) serving_completion = OpenAIServingChat( engine, models, response_role="assistant", - openai_serving_render=openai_serving_render, + online_renderer=online_renderer, chat_template=CHAT_TEMPLATE, chat_template_content_format="auto", request_logger=None, @@ -1396,9 +1395,7 @@ class TestServingChatWithHarmony: messages=messages, include_reasoning=include_reasoning, ) - input_messages, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req) - ) + input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req) verify_harmony_messages( input_messages, [ @@ -1429,8 +1426,8 @@ class TestServingChatWithHarmony: # Test the Harmony messages for the second turn's input req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages_2, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req_2) + input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony( + req_2 ) verify_harmony_messages( input_messages_2, @@ -1452,9 +1449,7 @@ class TestServingChatWithHarmony: {"role": "user", "content": "Hello"}, ] req = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req) - ) + input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req) verify_harmony_messages( input_messages, [ @@ -1478,9 +1473,7 @@ class TestServingChatWithHarmony: req = ChatCompletionRequest( model=MODEL_NAME, messages=messages, tools=weather_tools ) - input_messages, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req) - ) + input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req) verify_harmony_messages( input_messages, [ @@ -1503,9 +1496,7 @@ class TestServingChatWithHarmony: # Test the Harmony messages for the first turn's input req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools) - input_messages, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req) - ) + input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req) verify_harmony_messages( input_messages, [ @@ -1549,8 +1540,8 @@ class TestServingChatWithHarmony: # Test the Harmony messages for the second turn's input req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools) - input_messages_2, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req_2) + input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony( + req_2 ) verify_harmony_messages( input_messages_2, @@ -1588,9 +1579,7 @@ class TestServingChatWithHarmony: # Test the Harmony messages for the first turn's input req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools) - input_messages, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req) - ) + input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req) verify_harmony_messages( input_messages, [ @@ -1634,8 +1623,8 @@ class TestServingChatWithHarmony: # Test the Harmony messages for the second turn's input req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools) - input_messages_2, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req_2) + input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony( + req_2 ) verify_harmony_messages( input_messages_2, @@ -1686,8 +1675,8 @@ class TestServingChatWithHarmony: # Test the Harmony messages for the third turn's input req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools) - input_messages_3, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req_3) + input_messages_3, _ = serving_chat.online_renderer._make_request_with_harmony( + req_3 ) verify_harmony_messages( input_messages_3, @@ -1751,8 +1740,8 @@ class TestServingChatWithHarmony: # Test the Harmony messages for the fourth turn's input req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools) - input_messages_4, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req_4) + input_messages_4, _ = serving_chat.online_renderer._make_request_with_harmony( + req_4 ) verify_harmony_messages( input_messages_4, @@ -1802,9 +1791,7 @@ class TestServingChatWithHarmony: }, ] req = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req) - ) + input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req) verify_harmony_messages( input_messages, @@ -1835,9 +1822,7 @@ class TestServingChatWithHarmony: }, ] req = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req) - ) + input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req) verify_harmony_messages( input_messages, @@ -1866,9 +1851,7 @@ class TestServingChatWithHarmony: }, ] req = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages, _ = ( - serving_chat.openai_serving_render._make_request_with_harmony(req) - ) + input_messages, _ = serving_chat.online_renderer._make_request_with_harmony(req) verify_harmony_messages( input_messages, @@ -1898,14 +1881,14 @@ async def test_tool_choice_validation_without_parser(): engine_client=mock_engine, base_model_paths=BASE_MODEL_PATHS, ) - openai_serving_render = _build_serving_render(mock_engine, models.registry) + online_renderer = _build_online_renderer(mock_engine, models.registry) # Create serving_chat without tool_parser (enable_auto_tools=False) serving_chat = OpenAIServingChat( mock_engine, models, response_role="assistant", - openai_serving_render=openai_serving_render, + online_renderer=online_renderer, chat_template=CHAT_TEMPLATE, chat_template_content_format="auto", request_logger=None, @@ -1967,13 +1950,13 @@ async def test_streaming_n_gt1_independent_tool_parsers(): engine_client=mock_engine, base_model_paths=BASE_MODEL_PATHS, ) - openai_serving_render = _build_serving_render(mock_engine, models.registry) + online_renderer = _build_online_renderer(mock_engine, models.registry) serving_chat = OpenAIServingChat( mock_engine, models, response_role="assistant", - openai_serving_render=openai_serving_render, + online_renderer=online_renderer, chat_template=CHAT_TEMPLATE, chat_template_content_format="auto", request_logger=None, diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index 71a70a4d0e..9d2fedae36 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -14,9 +14,11 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.render.serving import OpenAIServingRender +from vllm.entrypoints.serve.render.serving import ServingRender from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers.hf import HfRenderer +from vllm.renderers.online_derenderer import OnlineDerenderer +from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.engine.async_llm import AsyncLLM @@ -77,10 +79,9 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion: engine_client=engine, base_model_paths=BASE_MODEL_PATHS, ) - serving_render = OpenAIServingRender( + online_renderer = OnlineRenderer( model_config=engine.model_config, renderer=engine.renderer, - model_registry=models.registry, request_logger=None, chat_template=None, chat_template_content_format="auto", @@ -88,7 +89,7 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion: return OpenAIServingCompletion( engine, models, - openai_serving_render=serving_render, + online_renderer=online_renderer, request_logger=None, ) @@ -158,7 +159,7 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution(): mock_engine.renderer = _build_renderer(mock_engine.model_config) serving_completion = _build_serving_completion(mock_engine) - serving_completion.openai_serving_render.preprocess_completion = AsyncMock( + serving_completion.online_renderer.preprocess_completion = AsyncMock( return_value=[{"prompt_token_ids": [1, 2, 3]}] ) @@ -171,13 +172,48 @@ async def test_openai_completion_keeps_mm_cache_for_engine_execution(): assert isinstance(result, list) assert ( - serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[ + serving_completion.online_renderer.preprocess_completion.call_args.kwargs[ "skip_mm_cache" ] is False ) +def _build_serving_render(engine: AsyncLLM) -> ServingRender: + models = OpenAIServingModels( + engine_client=engine, + base_model_paths=BASE_MODEL_PATHS, + ) + online_renderer = OnlineRenderer( + model_config=engine.model_config, + renderer=engine.renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + online_derenderer = OnlineDerenderer( + model_config=engine.model_config, + renderer=engine.renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + + serving_render = ServingRender(models, online_renderer, online_derenderer) + + async def _fake_preprocess_chat(*args, **kwargs): + # return conversation, engine_inputs + return ( + [{"role": "user", "content": "Test"}], + [{"prompt_token_ids": [1, 2, 3]}], + ) + + serving_render.online_renderer.preprocess_chat = AsyncMock( + side_effect=_fake_preprocess_chat + ) + return serving_render + + @pytest.mark.asyncio async def test_renderer_only_completion_request_skips_mm_cache(): mock_engine = MagicMock(spec=AsyncLLM) @@ -186,8 +222,9 @@ async def test_renderer_only_completion_request_skips_mm_cache(): mock_engine.input_processor = MagicMock() mock_engine.renderer = _build_renderer(mock_engine.model_config) - serving_completion = _build_serving_completion(mock_engine) - serving_completion.openai_serving_render.preprocess_completion = AsyncMock( + serving_render = _build_serving_render(mock_engine) + + serving_render.online_renderer.preprocess_completion = AsyncMock( return_value=[{"prompt_token_ids": [1, 2, 3]}] ) @@ -196,13 +233,11 @@ async def test_renderer_only_completion_request_skips_mm_cache(): prompt="Test prompt", ) - result = await serving_completion.openai_serving_render.render_completion_request( - request - ) + result = await serving_render.render_completion_request(request) assert isinstance(result, list) assert ( - serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[ + serving_render.online_renderer.preprocess_completion.call_args.kwargs[ "skip_mm_cache" ] is True diff --git a/tests/entrypoints/openai/completion/test_lora_resolvers.py b/tests/entrypoints/openai/completion/test_lora_resolvers.py index 6a0bec9251..30c2ce322f 100644 --- a/tests/entrypoints/openai/completion/test_lora_resolvers.py +++ b/tests/entrypoints/openai/completion/test_lora_resolvers.py @@ -14,10 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.lora.request import LoRARequest from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry from vllm.renderers.hf import HfRenderer +from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.engine.async_llm import AsyncLLM @@ -144,16 +144,15 @@ def mock_serving_setup(): base_model_paths=BASE_MODEL_PATHS, ) - serving_render = OpenAIServingRender( + online_renderer = OnlineRenderer( model_config=mock_engine.model_config, renderer=mock_engine.renderer, - model_registry=models.registry, request_logger=None, chat_template=None, chat_template_content_format="auto", ) serving_completion = OpenAIServingCompletion( - mock_engine, models, openai_serving_render=serving_render, request_logger=None + mock_engine, models, online_renderer=online_renderer, request_logger=None ) return mock_engine, serving_completion diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index b19abdff68..64d402663f 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -230,7 +230,7 @@ class TestInitializeToolSessions: instance = OpenAIServingResponses( engine_client=engine_client, models=models, - openai_serving_render=MagicMock(), + online_renderer=MagicMock(), request_logger=None, chat_template=None, chat_template_content_format="auto", @@ -316,7 +316,7 @@ class TestValidateGeneratorInput: instance = OpenAIServingResponses( engine_client=engine_client, models=models, - openai_serving_render=MagicMock(), + online_renderer=MagicMock(), request_logger=None, chat_template=None, chat_template_content_format="auto", @@ -379,7 +379,7 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): serving = OpenAIServingResponses( engine_client=engine_client, models=models, - openai_serving_render=MagicMock(), + online_renderer=MagicMock(), request_logger=None, chat_template=None, chat_template_content_format="auto", @@ -684,7 +684,7 @@ def _make_serving_instance_with_reasoning(): serving = OpenAIServingResponses( engine_client=engine_client, models=models, - openai_serving_render=MagicMock(), + online_renderer=MagicMock(), request_logger=None, chat_template=None, chat_template_content_format="auto", diff --git a/tests/entrypoints/pooling/embed/test_io_processor.py b/tests/entrypoints/pooling/embed/test_io_processor.py index 8f8f8faa8a..fbee91fc48 100644 --- a/tests/entrypoints/pooling/embed/test_io_processor.py +++ b/tests/entrypoints/pooling/embed/test_io_processor.py @@ -428,8 +428,8 @@ class TestPreProcessCohereOnline: handler._get_task_instruction_prefix = lambda _input_type: None handler._has_chat_template = lambda: False handler._preprocess_cmpl_online = preprocess_cmpl_online - handler._batch_render_chat = lambda *_args, **_kwargs: ( - pytest.fail("text-only request should not require chat rendering") + handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail( + "text-only request should not require chat rendering" ) handler._pre_process_cohere_online(ctx) @@ -448,8 +448,8 @@ class TestPreProcessCohereOnline: handler._get_task_instruction_prefix = lambda _input_type: "query: " handler._has_chat_template = lambda: False - handler._batch_render_chat = lambda *_args, **_kwargs: ( - pytest.fail("chat rendering should be skipped without a template") + handler._batch_render_chat = lambda *_args, **_kwargs: pytest.fail( + "chat rendering should be skipped without a template" ) handler._preprocess_cmpl_online = preprocess_cmpl @@ -485,8 +485,8 @@ class TestPreProcessCohereOnline: handler._get_task_instruction_prefix = lambda _input_type: "query: " handler._has_chat_template = lambda: True handler._batch_render_chat = batch_render_chat - handler._preprocess_cmpl_online = lambda *_args, **_kwargs: ( - pytest.fail("completion path should be skipped when a template exists") + handler._preprocess_cmpl_online = lambda *_args, **_kwargs: pytest.fail( + "completion path should be skipped when a template exists" ) handler._pre_process_cohere_online(ctx) diff --git a/tests/entrypoints/serve/disagg/test_generate_stream.py b/tests/entrypoints/serve/disagg/test_generate_stream.py index bd52863342..a31655e430 100644 --- a/tests/entrypoints/serve/disagg/test_generate_stream.py +++ b/tests/entrypoints/serve/disagg/test_generate_stream.py @@ -17,10 +17,10 @@ from vllm.entrypoints.serve.disagg.protocol import ( GenerateResponse, ) from vllm.entrypoints.serve.disagg.serving import ServingTokens -from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.logprobs import Logprob from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers import renderer_from_config +from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import SamplingParams from vllm.v1.engine.async_llm import AsyncLLM @@ -92,10 +92,9 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens: engine_client=engine, base_model_paths=BASE_MODEL_PATHS, ) - serving_render = OpenAIServingRender( + online_renderer = OnlineRenderer( model_config=engine.model_config, renderer=engine.renderer, - model_registry=models.registry, request_logger=None, chat_template=None, chat_template_content_format="auto", @@ -103,7 +102,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens: serving = ServingTokens( engine, models, - openai_serving_render=serving_render, + online_renderer=online_renderer, request_logger=None, **kwargs, ) @@ -111,7 +110,7 @@ def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens: async def _fake_preprocess(*args, **kwargs): return [{"prompt_token_ids": [1, 2, 3]}] - serving.openai_serving_render.preprocess_completion = AsyncMock( + serving.online_renderer.preprocess_completion = AsyncMock( side_effect=_fake_preprocess ) return serving @@ -199,9 +198,7 @@ async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution(): assert isinstance(response, GenerateResponse) assert ( - serving.openai_serving_render.preprocess_completion.call_args.kwargs[ - "skip_mm_cache" - ] + serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"] is True ) diff --git a/tests/entrypoints/serve/tokenize/test_serving_tokenization.py b/tests/entrypoints/serve/tokenize/test_serving_tokenization.py index 7f629afb1a..99267d8575 100644 --- a/tests/entrypoints/serve/tokenize/test_serving_tokenization.py +++ b/tests/entrypoints/serve/tokenize/test_serving_tokenization.py @@ -10,12 +10,12 @@ import pytest from vllm.config.multimodal import MultiModalConfig from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.entrypoints.serve.tokenize.protocol import ( TokenizeChatRequest, TokenizeCompletionRequest, ) from vllm.entrypoints.serve.tokenize.serving import ServingTokenization +from vllm.renderers.online_renderer import OnlineRenderer from vllm.v1.engine.async_llm import AsyncLLM MODEL_NAME = "openai-community/gpt2" @@ -63,18 +63,16 @@ def _build_serving_tokenization(engine: AsyncLLM) -> ServingTokenization: engine_client=engine, base_model_paths=BASE_MODEL_PATHS, ) - serving_render = OpenAIServingRender( + online_renderer = OnlineRenderer( model_config=engine.model_config, renderer=engine.renderer, - model_registry=models.registry, request_logger=None, chat_template=None, chat_template_content_format="auto", ) return ServingTokenization( models, - openai_serving_render=serving_render, - request_logger=None, + online_renderer=online_renderer, chat_template=None, chat_template_content_format="auto", ) @@ -89,7 +87,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path(): mock_engine.renderer = MagicMock() serving = _build_serving_tokenization(mock_engine) - serving.openai_serving_render.preprocess_chat = AsyncMock( + serving.online_renderer.preprocess_chat = AsyncMock( return_value=( [{"role": "user", "content": "Test"}], [{"prompt_token_ids": [1, 2, 3]}], @@ -105,7 +103,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path(): assert response.tokens == [1, 2, 3] assert ( - serving.openai_serving_render.preprocess_chat.call_args.kwargs["skip_mm_cache"] + serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"] is True ) @@ -119,7 +117,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path(): mock_engine.renderer = MagicMock() serving = _build_serving_tokenization(mock_engine) - serving.openai_serving_render.preprocess_completion = AsyncMock( + serving.online_renderer.preprocess_completion = AsyncMock( return_value=[{"prompt_token_ids": [1, 2, 3]}] ) @@ -132,8 +130,6 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path(): assert response.tokens == [1, 2, 3] assert ( - serving.openai_serving_render.preprocess_completion.call_args.kwargs[ - "skip_mm_cache" - ] + serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"] is True ) diff --git a/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py b/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py index 08553c6411..11c797c27c 100644 --- a/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py +++ b/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py @@ -99,8 +99,8 @@ async def test_non_streaming_cancel_advances_all_chunk_generators(): engine_client = SimpleNamespace( errored=False, generate=Mock( - side_effect=lambda *_args, **_kwargs: ( - _records_start_then_never_finishes(started_request_ids, _args[2]) + side_effect=lambda *_args, **_kwargs: _records_start_then_never_finishes( + started_request_ids, _args[2] ) ), abort=AsyncMock(), diff --git a/tests/v1/engine/test_async_llm.py b/tests/v1/engine/test_async_llm.py index 92de5a7e98..afb6e4c98b 100644 --- a/tests/v1/engine/test_async_llm.py +++ b/tests/v1/engine/test_async_llm.py @@ -512,12 +512,11 @@ async def test_header_dp_rank_argument(): ) # Create render serving instance (required by OpenAIServingChat) - from vllm.entrypoints.serve.render.serving import OpenAIServingRender + from vllm.renderers.online_renderer import OnlineRenderer - serving_render = OpenAIServingRender( + online_renderer = OnlineRenderer( model_config=engine.model_config, renderer=engine.renderer, - model_registry=models.registry, request_logger=None, chat_template=None, chat_template_content_format="auto", @@ -528,7 +527,7 @@ async def test_header_dp_rank_argument(): engine_client=engine, models=models, response_role="assistant", - openai_serving_render=serving_render, + online_renderer=online_renderer, chat_template=None, chat_template_content_format="auto", request_logger=None, diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 3d0151aefa..15550b262b 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -10,7 +10,7 @@ import logging import time import uuid from collections.abc import AsyncGenerator -from typing import TYPE_CHECKING, Any +from typing import Any import jinja2 from fastapi import Request @@ -48,9 +48,7 @@ from vllm.entrypoints.openai.engine.protocol import ( from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.utils.api_utils import sanitize_message from vllm.entrypoints.serve.utils.request_logger import RequestLogger - -if TYPE_CHECKING: - from vllm.entrypoints.serve.render.serving import OpenAIServingRender +from vllm.renderers.online_renderer import OnlineRenderer logger = logging.getLogger(__name__) @@ -111,7 +109,7 @@ class AnthropicServingMessages(OpenAIServingChat): models: OpenAIServingModels, response_role: str, *, - openai_serving_render: "OpenAIServingRender", + online_renderer: "OnlineRenderer", request_logger: RequestLogger | None, chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, @@ -127,7 +125,7 @@ class AnthropicServingMessages(OpenAIServingChat): engine_client=engine_client, models=models, response_role=response_role, - openai_serving_render=openai_serving_render, + online_renderer=online_renderer, request_logger=request_logger, chat_template=chat_template, chat_template_content_format=chat_template_content_format, diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index b1e6cea44f..38ecdec5ce 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -87,7 +87,7 @@ async def init_generate_state( tool_server = None resolved_chat_template = load_chat_template(args.chat_template) - # Render endpoints are always backed by OpenAIServingRender so that + # Render endpoints are always backed by OnlineRenderer so that # /v1/chat/completions/render and /v1/completions/render work on both # generate-mode and render-only servers. Created in init_app_state. @@ -95,7 +95,7 @@ async def init_generate_state( OpenAIServingResponses( engine_client, state.openai_serving_models, - state.openai_serving_render, + state.online_renderer, request_logger=request_logger, chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, @@ -116,7 +116,7 @@ async def init_generate_state( engine_client=engine_client, models=state.openai_serving_models, response_role=args.response_role, - openai_serving_render=state.openai_serving_render, + online_renderer=state.online_renderer, request_logger=request_logger, chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, @@ -146,7 +146,7 @@ async def init_generate_state( OpenAIServingCompletion( engine_client, state.openai_serving_models, - openai_serving_render=state.openai_serving_render, + online_renderer=state.online_renderer, request_logger=request_logger, return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_prompt_tokens_details=args.enable_prompt_tokens_details, @@ -160,7 +160,7 @@ async def init_generate_state( engine_client, state.openai_serving_models, args.response_role, - openai_serving_render=state.openai_serving_render, + online_renderer=state.online_renderer, request_logger=request_logger, chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, @@ -179,7 +179,7 @@ async def init_generate_state( ServingTokens( engine_client, state.openai_serving_models, - state.openai_serving_render, + state.online_renderer, request_logger=request_logger, return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_prompt_tokens_details=args.enable_prompt_tokens_details, diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index a16f522183..9fc4560adb 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -32,7 +32,7 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware -from vllm.entrypoints.serve.render.serving import OpenAIServingRender +from vllm.entrypoints.serve.render.serving import ServingRender from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.utils.api_utils import ( @@ -55,6 +55,8 @@ from vllm.entrypoints.serve.utils.server_utils import ( from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager +from vllm.renderers.online_derenderer import OnlineDerenderer +from vllm.renderers.online_renderer import OnlineRenderer from vllm.tasks import POOLING_TASKS, SupportedTask from vllm.tool_parsers import ToolParserManager from vllm.tracing import instrument @@ -360,10 +362,24 @@ async def init_app_state( ) await state.openai_serving_models.init_static_loras() - state.openai_serving_render = OpenAIServingRender( + state.online_renderer = OnlineRenderer( + model_config=engine_client.model_config, + renderer=engine_client.renderer, + request_logger=request_logger, + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + trust_request_chat_template=args.trust_request_chat_template, + enable_auto_tools=args.enable_auto_tool_choice, + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, + tool_parser=args.tool_call_parser, + reasoning_parser=args.structured_outputs_config.reasoning_parser, + default_chat_template_kwargs=args.default_chat_template_kwargs, + log_error_stack=args.log_error_stack, + ) + + state.online_derenderer = OnlineDerenderer( model_config=engine_client.model_config, renderer=engine_client.renderer, - model_registry=state.openai_serving_models.registry, request_logger=request_logger, chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, @@ -378,13 +394,19 @@ async def init_app_state( state.serving_tokenization = ServingTokenization( state.openai_serving_models, - state.openai_serving_render, + state.online_renderer, request_logger=request_logger, chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, default_chat_template_kwargs=args.default_chat_template_kwargs, trust_request_chat_template=args.trust_request_chat_template, ) + state.serving_render = ServingRender( + state.openai_serving_models, + state.online_renderer, + state.online_derenderer, + request_logger=request_logger, + ) if "generate" in supported_tasks: from vllm.entrypoints.generate.api_router import init_generate_state @@ -423,8 +445,8 @@ async def init_render_app_state( """ from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry - from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.renderers import renderer_from_config + from vllm.renderers.online_renderer import OnlineRenderer served_model_names = args.served_model_name or [args.model] model_registry = OpenAIModelRegistry( @@ -443,10 +465,24 @@ async def init_render_app_state( renderer = renderer_from_config(vllm_config) resolved_chat_template = load_chat_template(args.chat_template) - state.openai_serving_render = OpenAIServingRender( + state.online_renderer = OnlineRenderer( + model_config=vllm_config.model_config, + renderer=renderer, + request_logger=request_logger, + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + trust_request_chat_template=args.trust_request_chat_template, + enable_auto_tools=args.enable_auto_tool_choice, + exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, + tool_parser=args.tool_call_parser, + reasoning_parser=args.reasoning_parser, + default_chat_template_kwargs=args.default_chat_template_kwargs, + log_error_stack=args.log_error_stack, + ) + + state.online_derenderer = OnlineDerenderer( model_config=vllm_config.model_config, renderer=renderer, - model_registry=model_registry, request_logger=request_logger, chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, @@ -462,13 +498,19 @@ async def init_render_app_state( state.openai_serving_models = model_registry state.serving_tokenization = ServingTokenization( model_registry, - state.openai_serving_render, + state.online_renderer, request_logger=request_logger, chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, default_chat_template_kwargs=args.default_chat_template_kwargs, trust_request_chat_template=args.trust_request_chat_template, ) + state.serving_render = ServingRender( + model_registry, + state.online_renderer, + state.online_derenderer, + request_logger=request_logger, + ) state.vllm_config = vllm_config # Disable stats logging — there is no engine to poll. diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py index 96ed7dcb77..a0fc867050 100644 --- a/vllm/entrypoints/openai/chat_completion/batch_serving.py +++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py @@ -47,7 +47,7 @@ class OpenAIServingChatBatch(OpenAIServingChat): """Validate the model and preprocess a batched chat completion request. Performs engine-aware checks then delegates per-conversation - preprocessing to OpenAIServingRender, validating the chat template + preprocessing to OnlineRenderer, validating the chat template once for the whole batch. Returns: @@ -62,19 +62,19 @@ class OpenAIServingChatBatch(OpenAIServingChat): if self.engine_client.errored: raise self.engine_client.dead_error - render = self.openai_serving_render + renderer = self.online_renderer - if not render.use_harmony: + if not renderer.use_harmony: # Common case: validate the chat template once for the whole batch. - error_check_ret = render.validate_chat_template( + error_check_ret = renderer.validate_chat_template( request_chat_template=request.chat_template, chat_template_kwargs=request.chat_template_kwargs, - trust_request_chat_template=render.trust_request_chat_template, + trust_request_chat_template=renderer.trust_request_chat_template, ) if error_check_ret is not None: return error_check_ret - parser = render.parser + parser = renderer.parser tool_dicts: list[dict] | None = None all_conversations: list[list[ConversationMessage]] = [] @@ -82,17 +82,17 @@ class OpenAIServingChatBatch(OpenAIServingChat): for messages in request.messages: single_request = request.to_chat_completion_request(messages) - if render.use_harmony: - conversation, engine_prompts = render._make_request_with_harmony( + if renderer.use_harmony: + conversation, engine_prompts = renderer._make_request_with_harmony( single_request, should_include_tools=tool_dicts is not None ) else: - conversation, engine_prompts = await render.preprocess_chat( + conversation, engine_prompts = await renderer.preprocess_chat( single_request, messages, - default_template=render.chat_template, - default_template_content_format=render.chat_template_content_format, - default_template_kwargs=render.default_chat_template_kwargs, + default_template=renderer.chat_template, + default_template_content_format=renderer.chat_template_content_format, + default_template_kwargs=renderer.default_chat_template_kwargs, tool_dicts=tool_dicts, parser=parser, ) diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 911421029c..0b41c4d7fa 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -7,7 +7,7 @@ import time from collections.abc import AsyncGenerator, AsyncIterator from collections.abc import Sequence as GenericSequence from http import HTTPStatus -from typing import TYPE_CHECKING, Any, Final, cast +from typing import Any, Final, cast import numpy as np import pybase64 as base64 @@ -61,14 +61,12 @@ from vllm.outputs import RequestOutput from vllm.parser import ParserManager from vllm.parser.abstract_parser import Parser from vllm.renderers import ChatParams +from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.utils.collection_utils import as_list from vllm.utils.mistral import is_mistral_tool_parser -if TYPE_CHECKING: - from vllm.entrypoints.serve.render.serving import OpenAIServingRender - logger = init_logger(__name__) @@ -112,7 +110,7 @@ class OpenAIServingChat(OpenAIServing): models: OpenAIServingModels, response_role: str, *, - openai_serving_render: "OpenAIServingRender", + online_renderer: "OnlineRenderer", request_logger: RequestLogger | None, chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, @@ -135,7 +133,7 @@ class OpenAIServingChat(OpenAIServing): return_tokens_as_token_ids=return_tokens_as_token_ids, ) - self.openai_serving_render = openai_serving_render + self.online_renderer = online_renderer self.response_role = response_role self.chat_template = chat_template self.chat_template_content_format: Final = chat_template_content_format @@ -212,7 +210,7 @@ class OpenAIServingChat(OpenAIServing): """ Validate the model and preprocess a chat completion request. - Delegates preprocessing logic to OpenAIServingRender, adding the + Delegates preprocessing logic to OnlineRenderer, adding the engine-aware checks (LoRA model validation, engine health). Returns: @@ -230,7 +228,7 @@ class OpenAIServingChat(OpenAIServing): if self.engine_client.errored: raise self.engine_client.dead_error - return await self.openai_serving_render.render_chat(request) + return await self.online_renderer.render_chat(request) async def create_chat_completion( self, diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index fef1741351..43150191ac 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -6,7 +6,7 @@ import io import time from collections.abc import AsyncGenerator, AsyncIterator from collections.abc import Sequence as GenericSequence -from typing import TYPE_CHECKING, cast +from typing import cast import numpy as np import pybase64 as base64 @@ -41,14 +41,12 @@ from vllm.inputs import EngineInput from vllm.logger import init_logger from vllm.logprobs import Logprob from vllm.outputs import RequestOutput +from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.utils.async_utils import merge_async_iterators from vllm.utils.collection_utils import as_list -if TYPE_CHECKING: - from vllm.entrypoints.serve.render.serving import OpenAIServingRender - logger = init_logger(__name__) @@ -58,7 +56,7 @@ class OpenAIServingCompletion(OpenAIServing): engine_client: EngineClient, models: OpenAIServingModels, *, - openai_serving_render: "OpenAIServingRender", + online_renderer: "OnlineRenderer", request_logger: RequestLogger | None, return_tokens_as_token_ids: bool = False, enable_prompt_tokens_details: bool = False, @@ -71,7 +69,7 @@ class OpenAIServingCompletion(OpenAIServing): return_tokens_as_token_ids=return_tokens_as_token_ids, ) - self.openai_serving_render = openai_serving_render + self.online_renderer = online_renderer self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_force_include_usage = enable_force_include_usage @@ -90,7 +88,7 @@ class OpenAIServingCompletion(OpenAIServing): """ Validate the model and preprocess a completion request. - Delegates preprocessing logic to OpenAIServingRender, adding the + Delegates preprocessing logic to OnlineRenderer, adding the engine-aware checks (LoRA model validation, engine health). Returns: @@ -106,7 +104,7 @@ class OpenAIServingCompletion(OpenAIServing): if self.engine_client.errored: raise self.engine_client.dead_error - return await self.openai_serving_render.render_completion(request) + return await self.online_renderer.render_completion(request) async def create_completion( self, diff --git a/vllm/entrypoints/openai/engine/serving.py b/vllm/entrypoints/openai/engine/serving.py index d32fbad52a..70955b49af 100644 --- a/vllm/entrypoints/openai/engine/serving.py +++ b/vllm/entrypoints/openai/engine/serving.py @@ -13,12 +13,8 @@ from starlette.datastructures import Headers from vllm.engine.protocol import EngineClient from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.completion.protocol import ( - CompletionRequest, -) +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, GenerationError, diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 048efda9f6..0f37f3f0f3 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -92,7 +92,6 @@ from vllm.entrypoints.openai.responses.utils import ( extract_function_tool_names, extract_tool_types, ) -from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.entrypoints.serve.utils.api_utils import get_max_tokens from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.exceptions import VLLMValidationError @@ -103,6 +102,7 @@ from vllm.logprobs import SampleLogprobs from vllm.lora.request import LoRARequest from vllm.outputs import CompletionOutput from vllm.parser import Parser, ParserManager +from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.tokenizers import TokenizerLike from vllm.utils import random_uuid @@ -155,7 +155,7 @@ class OpenAIServingResponses(OpenAIServing): self, engine_client: EngineClient, models: OpenAIServingModels, - openai_serving_render: OpenAIServingRender, + online_renderer: OnlineRenderer, *, request_logger: RequestLogger | None, chat_template: str | None, @@ -177,7 +177,7 @@ class OpenAIServingResponses(OpenAIServing): return_tokens_as_token_ids=return_tokens_as_token_ids, ) - self.openai_serving_render = openai_serving_render + self.online_renderer = online_renderer self.chat_template = chat_template self.chat_template_content_format: Final = chat_template_content_format self.chat_template_kwargs = default_chat_template_kwargs or {} @@ -633,7 +633,7 @@ class OpenAIServingResponses(OpenAIServing): prev_response_output=prev_response.output if prev_response else None, ) chat_template_kwargs = self._effective_chat_template_kwargs(request) - _, engine_inputs = await self.openai_serving_render.preprocess_chat( + _, engine_inputs = await self.online_renderer.preprocess_chat( request, messages, default_template=self.chat_template, @@ -657,7 +657,7 @@ class OpenAIServingResponses(OpenAIServing): request_input=messages, ) chat_template_kwargs = self._effective_chat_template_kwargs(request) - _, engine_inputs = await self.openai_serving_render.preprocess_chat( + _, engine_inputs = await self.online_renderer.preprocess_chat( request, new_messages, default_template=chat_template, diff --git a/vllm/entrypoints/serve/disagg/api_router.py b/vllm/entrypoints/serve/disagg/api_router.py index 60b671d6e8..e5bd351e01 100644 --- a/vllm/entrypoints/serve/disagg/api_router.py +++ b/vllm/entrypoints/serve/disagg/api_router.py @@ -10,9 +10,7 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Respons from fastapi.responses import JSONResponse, StreamingResponse from vllm.engine.protocol import EngineClient -from vllm.entrypoints.openai.engine.protocol import ( - ErrorResponse, -) +from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.serve.disagg.protocol import ( GenerateRequest, GenerateResponse, diff --git a/vllm/entrypoints/serve/disagg/serving.py b/vllm/entrypoints/serve/disagg/serving.py index 5031627ea0..cbd6f83f23 100644 --- a/vllm/entrypoints/serve/disagg/serving.py +++ b/vllm/entrypoints/serve/disagg/serving.py @@ -36,7 +36,6 @@ from vllm.entrypoints.serve.disagg.protocol import ( GenerateResponseStreamChoice, GenerateStreamResponse, ) -from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.inputs import EngineInput, mm_input @@ -48,6 +47,7 @@ from vllm.multimodal.inputs import ( PlaceholderRange, ) from vllm.outputs import RequestOutput +from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.utils.collection_utils import as_list @@ -61,7 +61,7 @@ class ServingTokens(OpenAIServing): self, engine_client: EngineClient, models: OpenAIServingModels, - openai_serving_render: OpenAIServingRender, + online_renderer: OnlineRenderer, *, request_logger: RequestLogger | None, force_no_detokenize: bool = False, @@ -75,7 +75,7 @@ class ServingTokens(OpenAIServing): request_logger=request_logger, return_tokens_as_token_ids=return_tokens_as_token_ids, ) - self.openai_serving_render = openai_serving_render + self.online_renderer = online_renderer self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_log_outputs = enable_log_outputs self.force_no_detokenize = force_no_detokenize @@ -168,7 +168,7 @@ class ServingTokens(OpenAIServing): cache_salt=request.cache_salt, ) else: - (engine_input,) = await self.openai_serving_render.preprocess_completion( + (engine_input,) = await self.online_renderer.preprocess_completion( request, prompt_input=request.token_ids, prompt_embeds=None, diff --git a/vllm/entrypoints/serve/engine/typing.py b/vllm/entrypoints/serve/engine/typing.py index 253dfb8d90..8f0b7835da 100644 --- a/vllm/entrypoints/serve/engine/typing.py +++ b/vllm/entrypoints/serve/engine/typing.py @@ -15,7 +15,12 @@ from vllm.entrypoints.openai.completion.protocol import ( CompletionResponse, ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse +from vllm.entrypoints.serve.disagg.protocol import ( + DerenderChatRequest, + DerenderCompletionRequest, + GenerateRequest, + GenerateResponse, +) from vllm.entrypoints.serve.tokenize.protocol import ( DetokenizeRequest, TokenizeChatRequest, @@ -45,11 +50,17 @@ class RendererChatRequest(RendererRequest, Protocol): CompletionLikeRequest: TypeAlias = ( - CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest + CompletionRequest + | TokenizeCompletionRequest + | DetokenizeRequest + | DerenderCompletionRequest ) ChatLikeRequest: TypeAlias = ( - ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest + ChatCompletionRequest + | BatchChatCompletionRequest + | TokenizeChatRequest + | DerenderChatRequest ) SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest diff --git a/vllm/entrypoints/serve/render/api_router.py b/vllm/entrypoints/serve/render/api_router.py index 350260c188..3b3ad47612 100644 --- a/vllm/entrypoints/serve/render/api_router.py +++ b/vllm/entrypoints/serve/render/api_router.py @@ -19,7 +19,7 @@ from vllm.entrypoints.serve.disagg.protocol import ( DerenderCompletionRequest, GenerateRequest, ) -from vllm.entrypoints.serve.render.serving import OpenAIServingRender +from vllm.entrypoints.serve.render.serving import ServingRender from vllm.entrypoints.serve.utils.api_utils import validate_json_request from vllm.logger import init_logger @@ -28,8 +28,8 @@ logger = init_logger(__name__) router = APIRouter() -def render(request: Request) -> OpenAIServingRender | None: - return getattr(request.app.state, "openai_serving_render", None) +def render(request: Request) -> ServingRender | None: + return getattr(request.app.state, "serving_render", None) @router.post( diff --git a/vllm/entrypoints/serve/render/serving.py b/vllm/entrypoints/serve/render/serving.py index 612ff6d35e..42cf2460c4 100644 --- a/vllm/entrypoints/serve/render/serving.py +++ b/vllm/entrypoints/serve/render/serving.py @@ -1,44 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import time -from collections.abc import Sequence -from http import HTTPStatus -from typing import Any, cast +from typing import cast -from openai_harmony import Message as OpenAIMessage - -from vllm.config import ModelConfig -from vllm.entrypoints.chat_utils import ( - ChatTemplateContentFormatOption, - ConversationMessage, -) from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionLogProbs, ChatCompletionRequest, ChatCompletionResponse, - ChatCompletionResponseChoice, - ChatMessage, ) from vllm.entrypoints.openai.completion.protocol import ( - CompletionLogProbs, CompletionRequest, CompletionResponse, - CompletionResponseChoice, ) from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, - ToolCall, UsageInfo, ) -from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder -from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry -from vllm.entrypoints.openai.parser.harmony_utils import ( - build_harmony_preamble, - extract_instructions_from_messages, - parse_chat_inputs_to_harmony_messages, - render_for_completion, +from vllm.entrypoints.openai.models.serving import ( + OpenAIModelRegistry, + OpenAIServingModels, ) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item from vllm.entrypoints.serve.disagg.protocol import ( DerenderChatRequest, @@ -47,207 +27,49 @@ from vllm.entrypoints.serve.disagg.protocol import ( MultiModalFeatures, PlaceholderRangeInfo, ) +from vllm.entrypoints.serve.engine.serving import BaseServing from vllm.entrypoints.serve.utils.api_utils import get_max_tokens -from vllm.entrypoints.serve.utils.error_response import create_error_response from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.inputs import ( EngineInput, MultiModalHashes, MultiModalInput, MultiModalPlaceholders, - PromptType, - SingletonPrompt, - tokens_input, ) from vllm.logger import init_logger -from vllm.parser import Parser, ParserManager -from vllm.renderers import BaseRenderer, merge_kwargs from vllm.renderers.inputs.preprocess import ( extract_prompt_components, extract_prompt_len, - parse_model_prompt, - prompt_to_seq, ) -from vllm.tokenizers import TokenizerLike +from vllm.renderers.online_derenderer import OnlineDerenderer +from vllm.renderers.online_renderer import OnlineRenderer from vllm.utils import random_uuid -from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser -from vllm.utils.mistral import mt as _mt logger = init_logger(__name__) -def _parse_token_id_placeholder(token: str) -> int | None: - """Extract token ID from a 'token_id:N' placeholder string.""" - if not token.startswith("token_id:"): - return None - try: - return int(token[len("token_id:") :]) - except ValueError: - return None - - -def _correct_decoded_token( - token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike -) -> str: - """Use preceding tokens as context to fix U+FFFD from byte-fallback. - - Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py. - """ - max_ctx = min(len(context_token_ids), 4) - - for num_ctx in range(1, max_ctx + 1): - context = context_token_ids[-num_ctx:] - full_decoded = tokenizer.decode(context + [token_id]) - - if full_decoded.endswith("�"): - continue - - clean_end = len(context) - for j in range(len(context) - 1, -1, -1): - if tokenizer.decode([context[j]]).endswith("�"): - clean_end = j - else: - break - - clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else "" - - if full_decoded.startswith(clean_prefix): - return full_decoded[len(clean_prefix) :] - - common_len = 0 - for a, b in zip(clean_prefix, full_decoded): - if a != b: - break - common_len += 1 - return full_decoded[common_len:] - - return "" - - -def _resolve_logprobs( - logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike -) -> ChatCompletionLogProbs: - """Resolve token_id:N placeholders in a ChatCompletionLogProbs object.""" - if logprobs.content is None: - return logprobs - - context_token_ids: list[int] = [] - resolved_content = [] - - for entry in logprobs.content: - token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer) - sampled_id = _parse_token_id_placeholder(entry.token) - - if token_str.endswith("�") and sampled_id is not None: - token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer) - token_bytes = list(token_str.encode("utf-8")) - - resolved_top = [] - for top in entry.top_logprobs: - top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer) - top_id = _parse_token_id_placeholder(top.token) - if top_str.endswith("�") and top_id is not None: - top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer) - top_bytes = list(top_str.encode("utf-8")) - resolved_top.append( - top.model_copy(update={"token": top_str, "bytes": top_bytes}) - ) - - resolved_content.append( - entry.model_copy( - update={ - "token": token_str, - "bytes": token_bytes, - "top_logprobs": resolved_top, - } - ) - ) - - if sampled_id is not None: - context_token_ids.append(sampled_id) - - return ChatCompletionLogProbs(content=resolved_content) - - -def _convert_chat_logprobs_to_completion_logprobs( - logprobs: ChatCompletionLogProbs, -) -> CompletionLogProbs: - """Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs - (parallel flat lists) as required by the /v1/completions response schema.""" - if logprobs.content is None: - return CompletionLogProbs() - - tokens: list[str] = [] - token_logprobs: list[float | None] = [] - top_logprobs_list: list[dict[str, float] | None] = [] - text_offset: list[int] = [] - - offset = 0 - for entry in logprobs.content: - text_offset.append(offset) - tokens.append(entry.token) - token_logprobs.append(entry.logprob) - top_logprobs_list.append( - {t.token: t.logprob for t in entry.top_logprobs} - if entry.top_logprobs - else None - ) - offset += len(entry.token) - - return CompletionLogProbs( - text_offset=text_offset, - token_logprobs=token_logprobs, - tokens=tokens, - top_logprobs=top_logprobs_list, - ) - - -class OpenAIServingRender: +class ServingRender(BaseServing): def __init__( self, - model_config: ModelConfig, - renderer: BaseRenderer, - model_registry: OpenAIModelRegistry, + models: OpenAIServingModels | OpenAIModelRegistry, + online_renderer: "OnlineRenderer", + online_derenderer: "OnlineDerenderer", *, - request_logger: RequestLogger | None, - chat_template: str | None, - chat_template_content_format: ChatTemplateContentFormatOption, - trust_request_chat_template: bool = False, - enable_auto_tools: bool = False, - exclude_tools_when_tool_choice_none: bool = False, - tool_parser: str | None = None, - reasoning_parser: str | None = None, - default_chat_template_kwargs: dict[str, Any] | None = None, - log_error_stack: bool = False, + request_logger: RequestLogger | None = None, ) -> None: - self.model_config = model_config - self.renderer = renderer - self.model_registry = model_registry - self.request_logger = request_logger - self.chat_template = chat_template - self.chat_template_content_format: ChatTemplateContentFormatOption = ( - chat_template_content_format + super().__init__( + models=models, + model_config=online_renderer.model_config, + request_logger=request_logger, ) - self.trust_request_chat_template = trust_request_chat_template - self.enable_auto_tools = enable_auto_tools - self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none - self.use_harmony = model_config.hf_config.model_type == "gpt_oss" - self.parser: type[Parser] | None = ParserManager.get_parser( - tool_parser_name=tool_parser, - reasoning_parser_name=reasoning_parser, - enable_auto_tools=enable_auto_tools, - model_name=model_config.model, - is_harmony=self.use_harmony, - ) - self.default_chat_template_kwargs: dict[str, Any] = ( - default_chat_template_kwargs or {} - ) - self.log_error_stack = log_error_stack - self.supports_browsing = False - self.supports_code_interpreter = False - self.default_sampling_params = model_config.get_diff_sampling_param() - mc = model_config + self.online_renderer = online_renderer + self.online_derenderer = online_derenderer + + self.default_sampling_params = ( + online_renderer.model_config.get_diff_sampling_param() + ) + mc = online_renderer.model_config self.override_max_tokens = ( self.default_sampling_params.get("max_tokens") if mc.generation_config not in ("auto", "vllm") @@ -273,7 +95,7 @@ class OpenAIServingRender: "Beam search is not supported by the render endpoint" ) - result = await self.render_chat(request, skip_mm_cache=True) + result = await self.online_renderer.render_chat(request, skip_mm_cache=True) if isinstance(result, ErrorResponse): return result @@ -319,90 +141,6 @@ class OpenAIServingRender: priority=request.priority, ) - async def render_chat( - self, - request: ChatCompletionRequest, - *, - skip_mm_cache: bool = False, - ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse: - """Core preprocessing logic for chat requests (no model/engine check). - - Called directly by render_chat_request and delegated to by - OpenAIServingChat.render_chat_request after its engine-aware checks. - """ - tokenizer = self.renderer.tokenizer - - tool_parser = self.parser.tool_parser_cls if self.parser is not None else None - - if is_mistral_tokenizer(tokenizer): - # because of issues with pydantic we need to potentially - # re-serialize the tool_calls field of the request - _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type] - _mt.truncate_tool_call_ids(request) # type: ignore[arg-type] - _mt.validate_request_params(request) - - # Check if tool parsing is unavailable (common condition) - tool_parsing_unavailable = ( - tool_parser is None - and not is_mistral_tokenizer(tokenizer) - and not self.use_harmony - ) - - # Validate tool_choice when tool parsing is required but unavailable - if tool_parsing_unavailable and request.tool_choice not in ( - None, - "none", - ): - if request.tool_choice == "auto" and not self.enable_auto_tools: - # for hf tokenizers, "auto" tools requires - # --enable-auto-tool-choice and --tool-call-parser - return self.create_error_response( - '"auto" tool choice requires ' - "--enable-auto-tool-choice and --tool-call-parser to be set" - ) - elif request.tool_choice != "auto": - # "required" or named tool requires tool parser - return self.create_error_response( - f'tool_choice="{request.tool_choice}" requires ' - "--tool-call-parser to be set" - ) - - if request.tools is None or ( - request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none - ): - tool_dicts = None - else: - tool_dicts = [tool.model_dump() for tool in request.tools] - - if not self.use_harmony: - # Common case. - error_check_ret = self.validate_chat_template( - request_chat_template=request.chat_template, - chat_template_kwargs=request.chat_template_kwargs, - trust_request_chat_template=self.trust_request_chat_template, - ) - if error_check_ret is not None: - return error_check_ret - - conversation, engine_inputs = await self.preprocess_chat( - request, - request.messages, - default_template=self.chat_template, - default_template_content_format=self.chat_template_content_format, - default_template_kwargs=self.default_chat_template_kwargs, - tool_dicts=tool_dicts, - parser=self.parser, - skip_mm_cache=skip_mm_cache, - ) - else: - # For GPT-OSS. - should_include_tools = tool_dicts is not None - conversation, engine_inputs = self._make_request_with_harmony( - request, should_include_tools - ) - - return conversation, engine_inputs - async def render_completion_request( self, request: CompletionRequest, @@ -415,7 +153,9 @@ class OpenAIServingRender: error_check_ret = await self._check_model(request) if error_check_ret is not None: return error_check_ret - result = await self.render_completion(request, skip_mm_cache=True) + result = await self.online_renderer.render_completion( + request, skip_mm_cache=True + ) if isinstance(result, ErrorResponse): return result generate_requests: list[GenerateRequest] = [] @@ -459,37 +199,116 @@ class OpenAIServingRender: return generate_requests - async def render_completion( + async def derender_chat_response( self, - request: CompletionRequest, - *, - skip_mm_cache: bool = False, - ) -> list[EngineInput] | ErrorResponse: - """Core preprocessing logic for completion requests (no model/engine check). + request: DerenderChatRequest, + ) -> ChatCompletionResponse | ErrorResponse: + """Postprocess a GenerateResponse into a ChatCompletionResponse. - Called directly by render_completion_request and delegated to by - OpenAIServingCompletion.render_completion_request after its engine-aware checks. + Non-streaming only: expects the complete GenerateResponse with all + token IDs present. Uses ``parser.parse()`` for one-shot extraction. + + When ``request.chat_request`` is provided, the parser splits the + output into (reasoning, content, tool_calls). Otherwise falls + back to plain detokenization. """ - # Return error for unsupported features. - if request.suffix is not None: - return self.create_error_response("suffix is not currently supported") + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret - if request.echo and request.prompt_embeds is not None: - return self.create_error_response("Echo is unsupported with prompt embeds.") - - if request.prompt_logprobs is not None and request.prompt_embeds is not None: - return self.create_error_response( - "prompt_logprobs is not compatible with prompt embeds." + try: + choices = await self.online_derenderer.derender_chat( + request.generate_response, request.chat_request ) + except ValueError as exc: + return self.create_error_response(str(exc)) - engine_inputs = await self.preprocess_completion( - request, - prompt_input=request.prompt, - prompt_embeds=request.prompt_embeds, - skip_mm_cache=skip_mm_cache, + prompt_tokens = ( + request.prompt_tokens if request.prompt_tokens is not None else 0 + ) + gen = request.generate_response + completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids) + usage = UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, ) - return engine_inputs + logger.debug( + "derender_chat request_id=%s model=%s choices=%d completion_tokens=%d", + gen.request_id, + request.model, + len(choices), + completion_tokens, + ) + return ChatCompletionResponse( + id=gen.request_id, + model=request.model, + created=int(time.time()), + choices=choices, + usage=usage, + prompt_logprobs=gen.prompt_logprobs, + kv_transfer_params=gen.kv_transfer_params, + ) + + async def derender_completion_response( + self, + request: DerenderCompletionRequest, + ) -> CompletionResponse | ErrorResponse: + """Postprocess a list of GenerateResponses into a CompletionResponse. + + Non-streaming only. Mirrors the multi-prompt completions case: one + GenerateResponse per prompt, parallel to the list[GenerateRequest] + from /v1/completions/render. + """ + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + ( + choices, + total_prompt_tokens, + total_completion_tokens, + ) = await self.online_derenderer.derender_completion( + request.generate_responses, request.prompt_tokens + ) + + if not request.generate_responses: + return self.create_error_response("generate_responses must not be empty") + + first = request.generate_responses[0] + kv_params = first.kv_transfer_params + if any( + r.kv_transfer_params != kv_params for r in request.generate_responses[1:] + ): + logger.warning( + "derender_completion: kv_transfer_params differ across responses; " + "setting to None on the aggregated response" + ) + kv_params = None + + usage = UsageInfo( + prompt_tokens=total_prompt_tokens, + completion_tokens=total_completion_tokens, + total_tokens=total_prompt_tokens + total_completion_tokens, + ) + + logger.debug( + "derender_completion request_id=%s model=%s choices=%d" + " completion_tokens=%d", + first.request_id, + request.model, + len(choices), + total_completion_tokens, + ) + return CompletionResponse( + id=first.request_id, + model=request.model, + created=int(time.time()), + choices=choices, + usage=usage, + kv_transfer_params=kv_params, + ) @staticmethod def _extract_mm_features( @@ -529,440 +348,3 @@ class OpenAIServingRender: mm_placeholders=mm_placeholders, kwargs_data=kwargs_data, ) - - def _make_request_with_harmony( - self, - request: ChatCompletionRequest, - should_include_tools: bool = True, - ): - """Build Harmony (GPT-OSS) messages and engine prompt from a chat request.""" - messages: list[OpenAIMessage] = [] - - # because of issues with pydantic we need to potentially - # re-serialize the tool_calls field of the request - # for more info: see comment in `maybe_serialize_tool_calls` - _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type] - - chat_messages = list(request.messages) - instructions, chat_messages = extract_instructions_from_messages(chat_messages) - - # Add system message. - # NOTE: In Chat Completion API, browsing is enabled by default - # if the model supports it. TODO: Support browsing. - assert not self.supports_browsing - assert not self.supports_code_interpreter - if (reasoning_effort := request.reasoning_effort) == "none": - raise ValueError(f"Harmony does not support {reasoning_effort=}") - tools = request.tools if should_include_tools else None - messages.extend( - build_harmony_preamble( - instructions=instructions, - tools=tools, # type: ignore[arg-type] - reasoning_effort=reasoning_effort, - with_custom_tools=should_include_tools, - ) - ) - - # Add remaining conversation messages. - messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages)) - - # Render prompt token ids. - prompt_token_ids = render_for_completion(messages) - engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt) - - return messages, [engine_input] - - async def derender_chat_response( - self, - request: DerenderChatRequest, - ) -> ChatCompletionResponse | ErrorResponse: - """Postprocess a GenerateResponse into a ChatCompletionResponse. - - Non-streaming only: expects the complete GenerateResponse with all - token IDs present. Uses ``parser.parse()`` for one-shot extraction. - - When ``request.chat_request`` is provided, the parser splits the - output into (reasoning, content, tool_calls). Otherwise falls - back to plain detokenization. - """ - error_check_ret = await self._check_model(request) - if error_check_ret is not None: - return error_check_ret - - tokenizer = self.renderer.get_tokenizer() - gen = request.generate_response - chat_request = request.chat_request - choices: list[ChatCompletionResponseChoice] = [] - - try: - for choice in gen.choices: - if not choice.token_ids: - raise ValueError( - f"choice {choice.index} has empty or null token_ids" - ) - - resolved_logprobs = ( - _resolve_logprobs(choice.logprobs, tokenizer) - if choice.logprobs is not None - else None - ) - - if self.parser is not None and chat_request is not None: - # Parser path: decode with special tokens preserved - # so the parser can see markers like , - # , or Harmony channel tokens. - decoded_text = tokenizer.decode( - choice.token_ids, skip_special_tokens=False - ) - - chat_template_kwargs: dict[str, Any] = {} - if not self.use_harmony: - chat_template_kwargs = ( - chat_request.build_chat_params( - self.chat_template, - self.chat_template_content_format, - ) - .with_defaults(self.default_chat_template_kwargs) - .chat_template_kwargs - ) - - parser = self.parser( - tokenizer, - chat_request.tools, - chat_template_kwargs=chat_template_kwargs, - ) - reasoning, content, tool_calls = parser.parse( - decoded_text, - chat_request, - enable_auto_tools=self.enable_auto_tools, - model_output_token_ids=choice.token_ids, - ) - - if not getattr(chat_request, "include_reasoning", True): - reasoning = None - - tc_items = ( - [ - ToolCall( - id=random_uuid(), - function=tc, - ) - for tc in tool_calls - ] - if tool_calls - else [] - ) - - message = ChatMessage( - role="assistant", - reasoning=reasoning, - content=content, - tool_calls=tc_items, - ) - else: - # No parser: plain detokenization. - decoded_text = tokenizer.decode( - choice.token_ids, skip_special_tokens=True - ) - message = ChatMessage(role="assistant", content=decoded_text) - - choices.append( - ChatCompletionResponseChoice( - index=choice.index, - message=message, - logprobs=resolved_logprobs, - finish_reason=choice.finish_reason, - ) - ) - except ValueError as exc: - return self.create_error_response(str(exc)) - - prompt_tokens = ( - request.prompt_tokens if request.prompt_tokens is not None else 0 - ) - completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids) - usage = UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ) - - logger.debug( - "derender_chat request_id=%s model=%s choices=%d completion_tokens=%d", - gen.request_id, - request.model, - len(choices), - completion_tokens, - ) - return ChatCompletionResponse( - id=gen.request_id, - model=request.model, - created=int(time.time()), - choices=choices, - usage=usage, - prompt_logprobs=gen.prompt_logprobs, - kv_transfer_params=gen.kv_transfer_params, - ) - - async def derender_completion_response( - self, - request: DerenderCompletionRequest, - ) -> CompletionResponse | ErrorResponse: - """Postprocess a list of GenerateResponses into a CompletionResponse. - - Non-streaming only. Mirrors the multi-prompt completions case: one - GenerateResponse per prompt, parallel to the list[GenerateRequest] - from /v1/completions/render. - """ - error_check_ret = await self._check_model(request) - if error_check_ret is not None: - return error_check_ret - - n = len(request.generate_responses) - prompt_tokens_list: list[int] = ( - request.prompt_tokens if request.prompt_tokens is not None else [0] * n - ) - - tokenizer = self.renderer.get_tokenizer() - choices: list[CompletionResponseChoice] = [] - total_prompt_tokens = 0 - total_completion_tokens = 0 - index = 0 - - for gen, pt in zip(request.generate_responses, prompt_tokens_list): - for choice in gen.choices: - if not choice.token_ids: - return self.create_error_response( - f"choice {choice.index} in response {gen.request_id} " - "has empty or null token_ids" - ) - decoded_text = tokenizer.decode( - choice.token_ids, skip_special_tokens=True - ) - completion_logprobs = None - if choice.logprobs is not None: - resolved = _resolve_logprobs(choice.logprobs, tokenizer) - completion_logprobs = _convert_chat_logprobs_to_completion_logprobs( - resolved - ) - choices.append( - CompletionResponseChoice( - index=index, - text=decoded_text, - finish_reason=choice.finish_reason, - logprobs=completion_logprobs, - ) - ) - total_completion_tokens += len(choice.token_ids) - index += 1 - total_prompt_tokens += pt - - if not request.generate_responses: - return self.create_error_response("generate_responses must not be empty") - - first = request.generate_responses[0] - kv_params = first.kv_transfer_params - if any( - r.kv_transfer_params != kv_params for r in request.generate_responses[1:] - ): - logger.warning( - "derender_completion: kv_transfer_params differ across responses; " - "setting to None on the aggregated response" - ) - kv_params = None - - usage = UsageInfo( - prompt_tokens=total_prompt_tokens, - completion_tokens=total_completion_tokens, - total_tokens=total_prompt_tokens + total_completion_tokens, - ) - - logger.debug( - "derender_completion request_id=%s model=%s choices=%d" - " completion_tokens=%d", - first.request_id, - request.model, - len(choices), - total_completion_tokens, - ) - return CompletionResponse( - id=first.request_id, - model=request.model, - created=int(time.time()), - choices=choices, - usage=usage, - kv_transfer_params=kv_params, - ) - - def create_error_response( - self, - message: str | Exception, - err_type: str = "BadRequestError", - status_code: HTTPStatus = HTTPStatus.BAD_REQUEST, - param: str | None = None, - ) -> ErrorResponse: - return create_error_response(message, err_type, status_code, param) - - async def _check_model( - self, - request: Any, - ) -> ErrorResponse | None: - return await self.model_registry.check_model(request.model) - - def validate_chat_template( - self, - request_chat_template: str | None, - chat_template_kwargs: dict[str, Any] | None, - trust_request_chat_template: bool, - ) -> ErrorResponse | None: - """Copied from OpenAIServing._validate_chat_template.""" - if not trust_request_chat_template and ( - request_chat_template is not None - or ( - chat_template_kwargs - and chat_template_kwargs.get("chat_template") is not None - ) - ): - return self.create_error_response( - "Chat template is passed with request, but " - "--trust-request-chat-template is not set. " - "Refused request with untrusted chat template." - ) - return None - - async def preprocess_completion( - self, - request: Any, - prompt_input: str | list[str] | list[int] | list[list[int]] | None, - prompt_embeds: bytes | list[bytes] | None, - *, - skip_mm_cache: bool = False, - ) -> list[EngineInput]: - """Copied from OpenAIServing._preprocess_completion.""" - prompts = list[SingletonPrompt | bytes]() - if prompt_embeds is not None: # embeds take higher priority - prompts.extend(prompt_to_seq(prompt_embeds)) - if prompt_input is not None: - prompts.extend(prompt_to_seq(prompt_input)) - return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache) - - async def preprocess_cmpl( - self, - request: Any, - prompts: Sequence[PromptType | bytes], - *, - skip_mm_cache: bool = False, - ) -> list[EngineInput]: - """Copied from OpenAIServing._preprocess_cmpl.""" - renderer = self.renderer - model_config = self.model_config - - parsed_prompts = [ - ( - prompt - if isinstance(prompt, bytes) - else parse_model_prompt(model_config, prompt) - ) - for prompt in prompts - ] - tok_params = request.build_tok_params(model_config) - - return await renderer.render_cmpl_async( - parsed_prompts, - tok_params, - prompt_extras={ - k: v - for k in ("mm_processor_kwargs", "cache_salt") - if (v := getattr(request, k, None)) is not None - }, - skip_mm_cache=skip_mm_cache, - ) - - async def preprocess_chat( - self, - request: Any, - messages: list[Any], - default_template: str | None, - default_template_content_format: ChatTemplateContentFormatOption, - default_template_kwargs: dict[str, Any] | None, - tool_dicts: list[dict[str, Any]] | None = None, - parser: type[Parser] | None = None, - *, - skip_mm_cache: bool = False, - ) -> tuple[list[ConversationMessage], list[EngineInput]]: - """Copied from OpenAIServing._preprocess_chat.""" - renderer = self.renderer - mm_config = self.model_config.multimodal_config - - default_template_kwargs = merge_kwargs( - default_template_kwargs, - dict( - tools=tool_dicts, - tokenize=( - is_mistral_tokenizer(renderer.tokenizer) - or self.model_config.enable_prompt_embeds - ), - ), - ) - - tok_params = request.build_tok_params(self.model_config) - chat_params = request.build_chat_params( - default_template, default_template_content_format - ).with_defaults( - default_template_kwargs, - default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None), - default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), - ) - - (conversation,), (engine_input,) = await renderer.render_chat_async( - [messages], - chat_params, - tok_params, - prompt_extras={ - k: v - for k in ("mm_processor_kwargs", "cache_salt") - if (v := getattr(request, k, None)) is not None - }, - skip_mm_cache=skip_mm_cache, - ) - - # tool parsing is done only if a tool_parser has been set and if - # tool_choice is not "none" (if tool_choice is "none" but a tool_parser - # is set, we want to prevent parsing a tool_call hallucinated by the LLM - # - # Exception: Mistral grammar-capable tokenizers always call - # adjust_request — even for tool_choice="none" — so that the grammar - # factory can prevent special-token leakage. - if parser is not None: - tokenizer = renderer.get_tokenizer() - tool_parser = parser.tool_parser_cls - tool_choice = getattr(request, "tool_choice", "none") - is_mistral_grammar_eligible = ( - tool_parser is not None - and is_mistral_tool_parser(tool_parser) - and is_mistral_tokenizer(tokenizer) - and tokenizer.supports_grammar - ) - should_adjust_request = ( - parser.reasoning_parser_cls is not None - or tool_choice != "none" - or is_mistral_grammar_eligible - ) - if should_adjust_request: - if not isinstance(request, ChatCompletionRequest | ResponsesRequest): - msg = ( - "Tool usage is only supported " - "for Chat Completions API or Responses API requests, " - f"but got {type(request).__name__}" - ) - raise NotImplementedError(msg) - request = parser( - tokenizer, - request.tools, - model_config=self.model_config, - chat_template_kwargs=chat_params.chat_template_kwargs, - ).adjust_request( - request=request, - ) - - return conversation, [engine_input] diff --git a/vllm/entrypoints/serve/tokenize/serving.py b/vllm/entrypoints/serve/tokenize/serving.py index d898412e2a..12072c3f6b 100644 --- a/vllm/entrypoints/serve/tokenize/serving.py +++ b/vllm/entrypoints/serve/tokenize/serving.py @@ -12,7 +12,6 @@ from vllm.entrypoints.openai.models.serving import ( OpenAIServingModels, ) from vllm.entrypoints.serve.engine.serving import BaseServing -from vllm.entrypoints.serve.render.serving import OpenAIServingRender from vllm.entrypoints.serve.tokenize.protocol import ( DetokenizeRequest, DetokenizeResponse, @@ -24,6 +23,7 @@ from vllm.entrypoints.serve.tokenize.protocol import ( from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.inputs import TokensPrompt, tokens_input from vllm.logger import init_logger +from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) @@ -33,22 +33,22 @@ class ServingTokenization(BaseServing): def __init__( self, models: OpenAIServingModels | OpenAIModelRegistry, - openai_serving_render: OpenAIServingRender, + online_renderer: OnlineRenderer, *, - request_logger: RequestLogger | None, chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, default_chat_template_kwargs: dict[str, Any] | None = None, trust_request_chat_template: bool = False, + request_logger: RequestLogger | None = None, ) -> None: super().__init__( models=models, - model_config=openai_serving_render.model_config, + model_config=online_renderer.model_config, request_logger=request_logger, ) - self.renderer = openai_serving_render.renderer - self.openai_serving_render = openai_serving_render + self.renderer = online_renderer.renderer + self.online_renderer = online_renderer self.chat_template = chat_template self.chat_template_content_format: Final = chat_template_content_format self.default_chat_template_kwargs = default_chat_template_kwargs or {} @@ -73,7 +73,7 @@ class ServingTokenization(BaseServing): if request.tools is None else [tool.model_dump() for tool in request.tools] ) - error_check_ret = self.openai_serving_render.validate_chat_template( + error_check_ret = self.online_renderer.validate_chat_template( request_chat_template=request.chat_template, chat_template_kwargs=request.chat_template_kwargs, trust_request_chat_template=self.trust_request_chat_template, @@ -81,7 +81,7 @@ class ServingTokenization(BaseServing): if error_check_ret is not None: return error_check_ret - _, engine_inputs = await self.openai_serving_render.preprocess_chat( + _, engine_inputs = await self.online_renderer.preprocess_chat( request, request.messages, default_template=self.chat_template, @@ -91,7 +91,7 @@ class ServingTokenization(BaseServing): skip_mm_cache=True, ) else: - engine_inputs = await self.openai_serving_render.preprocess_completion( + engine_inputs = await self.online_renderer.preprocess_completion( request, prompt_input=request.prompt, prompt_embeds=None, diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py new file mode 100644 index 0000000000..91d03bbe81 --- /dev/null +++ b/vllm/renderers/online_derenderer.py @@ -0,0 +1,334 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + +from vllm.config import ModelConfig +from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionLogProbs, + ChatCompletionRequest, + ChatCompletionResponseChoice, + ChatMessage, +) +from vllm.entrypoints.openai.completion.protocol import ( + CompletionLogProbs, + CompletionResponseChoice, +) +from vllm.entrypoints.openai.engine.protocol import ToolCall +from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder +from vllm.entrypoints.serve.disagg.protocol import GenerateResponse +from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.logger import init_logger +from vllm.parser import Parser, ParserManager +from vllm.renderers import BaseRenderer +from vllm.tokenizers import TokenizerLike +from vllm.utils import random_uuid + +logger = init_logger(__name__) + + +class OnlineDerenderer: + def __init__( + self, + model_config: ModelConfig, + renderer: BaseRenderer, + *, + request_logger: RequestLogger | None, + chat_template: str | None, + chat_template_content_format: ChatTemplateContentFormatOption, + trust_request_chat_template: bool = False, + enable_auto_tools: bool = False, + exclude_tools_when_tool_choice_none: bool = False, + tool_parser: str | None = None, + reasoning_parser: str | None = None, + default_chat_template_kwargs: dict[str, Any] | None = None, + log_error_stack: bool = False, + ) -> None: + self.model_config = model_config + self.renderer = renderer + self.request_logger = request_logger + + self.enable_auto_tools = enable_auto_tools + self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none + self.use_harmony = model_config.hf_config.model_type == "gpt_oss" + self.parser: type[Parser] | None = ParserManager.get_parser( + tool_parser_name=tool_parser, + reasoning_parser_name=reasoning_parser, + enable_auto_tools=enable_auto_tools, + model_name=model_config.model, + is_harmony=self.use_harmony, + ) + + self.chat_template = chat_template + self.chat_template_content_format: ChatTemplateContentFormatOption = ( + chat_template_content_format + ) + self.default_chat_template_kwargs: dict[str, Any] = ( + default_chat_template_kwargs or {} + ) + self.trust_request_chat_template = trust_request_chat_template + + self.log_error_stack = log_error_stack + self.supports_browsing = False + self.supports_code_interpreter = False + + async def derender_chat( + self, + generate_response: GenerateResponse, + chat_request: ChatCompletionRequest | None = None, + ) -> list[ChatCompletionResponseChoice]: + tokenizer = self.renderer.get_tokenizer() + choices: list[ChatCompletionResponseChoice] = [] + + for choice in generate_response.choices: + if not choice.token_ids: + raise ValueError(f"choice {choice.index} has empty or null token_ids") + + resolved_logprobs = ( + _resolve_logprobs(choice.logprobs, tokenizer) + if choice.logprobs is not None + else None + ) + + if self.parser is not None and chat_request is not None: + # Parser path: decode with special tokens preserved + # so the parser can see markers like , + # , or Harmony channel tokens. + decoded_text = tokenizer.decode( + choice.token_ids, skip_special_tokens=False + ) + + chat_template_kwargs: dict[str, Any] = {} + if not self.use_harmony: + chat_template_kwargs = ( + chat_request.build_chat_params( + self.chat_template, + self.chat_template_content_format, + ) + .with_defaults(self.default_chat_template_kwargs) + .chat_template_kwargs + ) + + parser = self.parser( + tokenizer, + chat_request.tools, + chat_template_kwargs=chat_template_kwargs, + ) + reasoning, content, tool_calls = parser.parse( + decoded_text, + chat_request, + enable_auto_tools=self.enable_auto_tools, + model_output_token_ids=choice.token_ids, + ) + + if not getattr(chat_request, "include_reasoning", True): + reasoning = None + + tc_items = ( + [ + ToolCall( + id=random_uuid(), + function=tc, + ) + for tc in tool_calls + ] + if tool_calls + else [] + ) + + message = ChatMessage( + role="assistant", + reasoning=reasoning, + content=content, + tool_calls=tc_items, + ) + else: + # No parser: plain detokenization. + decoded_text = tokenizer.decode( + choice.token_ids, skip_special_tokens=True + ) + message = ChatMessage(role="assistant", content=decoded_text) + + choices.append( + ChatCompletionResponseChoice( + index=choice.index, + message=message, + logprobs=resolved_logprobs, + finish_reason=choice.finish_reason, + ) + ) + + return choices + + async def derender_completion( + self, + generate_responses: list[GenerateResponse], + prompt_tokens: list[int] | None = None, + ) -> tuple[list[CompletionResponseChoice], int, int]: + n = len(generate_responses) + prompt_tokens_list: list[int] = ( + prompt_tokens if prompt_tokens is not None else [0] * n + ) + + tokenizer = self.renderer.get_tokenizer() + choices: list[CompletionResponseChoice] = [] + total_prompt_tokens = 0 + total_completion_tokens = 0 + index = 0 + + for gen, pt in zip(generate_responses, prompt_tokens_list): + for choice in gen.choices: + if not choice.token_ids: + raise ValueError( + f"choice {choice.index} in response {gen.request_id} " + "has empty or null token_ids" + ) + + decoded_text = tokenizer.decode( + choice.token_ids, skip_special_tokens=True + ) + completion_logprobs = None + if choice.logprobs is not None: + resolved = _resolve_logprobs(choice.logprobs, tokenizer) + completion_logprobs = _convert_chat_logprobs_to_completion_logprobs( + resolved + ) + choices.append( + CompletionResponseChoice( + index=index, + text=decoded_text, + finish_reason=choice.finish_reason, + logprobs=completion_logprobs, + ) + ) + total_completion_tokens += len(choice.token_ids) + index += 1 + total_prompt_tokens += pt + + return choices, total_prompt_tokens, total_completion_tokens + + +def _parse_token_id_placeholder(token: str) -> int | None: + """Extract token ID from a 'token_id:N' placeholder string.""" + if not token.startswith("token_id:"): + return None + try: + return int(token[len("token_id:") :]) + except ValueError: + return None + + +def _correct_decoded_token( + token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike +) -> str: + """Use preceding tokens as context to fix U+FFFD from byte-fallback. + + Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py. + """ + max_ctx = min(len(context_token_ids), 4) + + for num_ctx in range(1, max_ctx + 1): + context = context_token_ids[-num_ctx:] + full_decoded = tokenizer.decode(context + [token_id]) + + if full_decoded.endswith("�"): + continue + + clean_end = len(context) + for j in range(len(context) - 1, -1, -1): + if tokenizer.decode([context[j]]).endswith("�"): + clean_end = j + else: + break + + clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else "" + + if full_decoded.startswith(clean_prefix): + return full_decoded[len(clean_prefix) :] + + common_len = 0 + for a, b in zip(clean_prefix, full_decoded): + if a != b: + break + common_len += 1 + return full_decoded[common_len:] + + return "" + + +def _resolve_logprobs( + logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike +) -> ChatCompletionLogProbs: + """Resolve token_id:N placeholders in a ChatCompletionLogProbs object.""" + if logprobs.content is None: + return logprobs + + context_token_ids: list[int] = [] + resolved_content = [] + + for entry in logprobs.content: + token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer) + sampled_id = _parse_token_id_placeholder(entry.token) + + if token_str.endswith("�") and sampled_id is not None: + token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer) + token_bytes = list(token_str.encode("utf-8")) + + resolved_top = [] + for top in entry.top_logprobs: + top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer) + top_id = _parse_token_id_placeholder(top.token) + if top_str.endswith("�") and top_id is not None: + top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer) + top_bytes = list(top_str.encode("utf-8")) + resolved_top.append( + top.model_copy(update={"token": top_str, "bytes": top_bytes}) + ) + + resolved_content.append( + entry.model_copy( + update={ + "token": token_str, + "bytes": token_bytes, + "top_logprobs": resolved_top, + } + ) + ) + + if sampled_id is not None: + context_token_ids.append(sampled_id) + + return ChatCompletionLogProbs(content=resolved_content) + + +def _convert_chat_logprobs_to_completion_logprobs( + logprobs: ChatCompletionLogProbs, +) -> CompletionLogProbs: + """Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs + (parallel flat lists) as required by the /v1/completions response schema.""" + if logprobs.content is None: + return CompletionLogProbs() + + tokens: list[str] = [] + token_logprobs: list[float | None] = [] + top_logprobs_list: list[dict[str, float] | None] = [] + text_offset: list[int] = [] + + offset = 0 + for entry in logprobs.content: + text_offset.append(offset) + tokens.append(entry.token) + token_logprobs.append(entry.logprob) + top_logprobs_list.append( + {t.token: t.logprob for t in entry.top_logprobs} + if entry.top_logprobs + else None + ) + offset += len(entry.token) + + return CompletionLogProbs( + text_offset=text_offset, + token_logprobs=token_logprobs, + tokens=tokens, + top_logprobs=top_logprobs_list, + ) diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py new file mode 100644 index 0000000000..4e188336cb --- /dev/null +++ b/vllm/renderers/online_renderer.py @@ -0,0 +1,417 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Sequence +from http import HTTPStatus +from typing import Any + +from openai_harmony import Message as OpenAIMessage + +from vllm.config import ModelConfig +from vllm.entrypoints.chat_utils import ( + ChatTemplateContentFormatOption, + ConversationMessage, +) +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, +) +from vllm.entrypoints.openai.completion.protocol import ( + CompletionRequest, +) +from vllm.entrypoints.openai.engine.protocol import ErrorResponse +from vllm.entrypoints.openai.parser.harmony_utils import ( + build_harmony_preamble, + extract_instructions_from_messages, + parse_chat_inputs_to_harmony_messages, + render_for_completion, +) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.entrypoints.serve.utils.error_response import create_error_response +from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.inputs import ( + EngineInput, + PromptType, + SingletonPrompt, + tokens_input, +) +from vllm.logger import init_logger +from vllm.parser import Parser, ParserManager +from vllm.renderers import BaseRenderer, merge_kwargs +from vllm.renderers.inputs.preprocess import ( + parse_model_prompt, + prompt_to_seq, +) +from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser +from vllm.utils.mistral import mt as _mt + +logger = init_logger(__name__) + + +class OnlineRenderer: + def __init__( + self, + model_config: ModelConfig, + renderer: BaseRenderer, + *, + request_logger: RequestLogger | None, + chat_template: str | None, + chat_template_content_format: ChatTemplateContentFormatOption, + trust_request_chat_template: bool = False, + enable_auto_tools: bool = False, + exclude_tools_when_tool_choice_none: bool = False, + tool_parser: str | None = None, + reasoning_parser: str | None = None, + default_chat_template_kwargs: dict[str, Any] | None = None, + log_error_stack: bool = False, + ) -> None: + self.model_config = model_config + self.renderer = renderer + self.request_logger = request_logger + + self.enable_auto_tools = enable_auto_tools + self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none + self.use_harmony = model_config.hf_config.model_type == "gpt_oss" + self.parser: type[Parser] | None = ParserManager.get_parser( + tool_parser_name=tool_parser, + reasoning_parser_name=reasoning_parser, + enable_auto_tools=enable_auto_tools, + model_name=model_config.model, + is_harmony=self.use_harmony, + ) + + self.chat_template = chat_template + self.chat_template_content_format: ChatTemplateContentFormatOption = ( + chat_template_content_format + ) + self.default_chat_template_kwargs: dict[str, Any] = ( + default_chat_template_kwargs or {} + ) + self.trust_request_chat_template = trust_request_chat_template + + self.log_error_stack = log_error_stack + self.supports_browsing = False + self.supports_code_interpreter = False + + async def render_chat( + self, + request: ChatCompletionRequest, + *, + skip_mm_cache: bool = False, + ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse: + """Core preprocessing logic for chat requests (no model/engine check). + + Called directly by render_chat_request and delegated to by + OpenAIServingChat.render_chat_request after its engine-aware checks. + """ + tokenizer = self.renderer.tokenizer + + tool_parser = self.parser.tool_parser_cls if self.parser is not None else None + + if is_mistral_tokenizer(tokenizer): + # because of issues with pydantic we need to potentially + # re-serialize the tool_calls field of the request + _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type] + _mt.truncate_tool_call_ids(request) # type: ignore[arg-type] + _mt.validate_request_params(request) + + # Check if tool parsing is unavailable (common condition) + tool_parsing_unavailable = ( + tool_parser is None + and not is_mistral_tokenizer(tokenizer) + and not self.use_harmony + ) + + # Validate tool_choice when tool parsing is required but unavailable + if tool_parsing_unavailable and request.tool_choice not in ( + None, + "none", + ): + if request.tool_choice == "auto" and not self.enable_auto_tools: + # for hf tokenizers, "auto" tools requires + # --enable-auto-tool-choice and --tool-call-parser + return self.create_error_response( + '"auto" tool choice requires ' + "--enable-auto-tool-choice and --tool-call-parser to be set" + ) + elif request.tool_choice != "auto": + # "required" or named tool requires tool parser + return self.create_error_response( + f'tool_choice="{request.tool_choice}" requires ' + "--tool-call-parser to be set" + ) + + if request.tools is None or ( + request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none + ): + tool_dicts = None + else: + tool_dicts = [tool.model_dump() for tool in request.tools] + + if not self.use_harmony: + # Common case. + error_check_ret = self.validate_chat_template( + request_chat_template=request.chat_template, + chat_template_kwargs=request.chat_template_kwargs, + trust_request_chat_template=self.trust_request_chat_template, + ) + if error_check_ret is not None: + return error_check_ret + + conversation, engine_inputs = await self.preprocess_chat( + request, + request.messages, + default_template=self.chat_template, + default_template_content_format=self.chat_template_content_format, + default_template_kwargs=self.default_chat_template_kwargs, + tool_dicts=tool_dicts, + parser=self.parser, + skip_mm_cache=skip_mm_cache, + ) + else: + # For GPT-OSS. + should_include_tools = tool_dicts is not None + conversation, engine_inputs = self._make_request_with_harmony( + request, should_include_tools + ) + + return conversation, engine_inputs + + def _make_request_with_harmony( + self, + request: ChatCompletionRequest, + should_include_tools: bool = True, + ): + """Build Harmony (GPT-OSS) messages and engine prompt from a chat request.""" + messages: list[OpenAIMessage] = [] + + # because of issues with pydantic we need to potentially + # re-serialize the tool_calls field of the request + # for more info: see comment in `maybe_serialize_tool_calls` + _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type] + + chat_messages = list(request.messages) + instructions, chat_messages = extract_instructions_from_messages(chat_messages) + + # Add system message. + # NOTE: In Chat Completion API, browsing is enabled by default + # if the model supports it. TODO: Support browsing. + assert not self.supports_browsing + assert not self.supports_code_interpreter + if (reasoning_effort := request.reasoning_effort) == "none": + raise ValueError(f"Harmony does not support {reasoning_effort=}") + tools = request.tools if should_include_tools else None + messages.extend( + build_harmony_preamble( + instructions=instructions, + tools=tools, # type: ignore[arg-type] + reasoning_effort=reasoning_effort, + with_custom_tools=should_include_tools, + ) + ) + + # Add remaining conversation messages. + messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages)) + + # Render prompt token ids. + prompt_token_ids = render_for_completion(messages) + engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt) + + return messages, [engine_input] + + async def render_completion( + self, + request: CompletionRequest, + *, + skip_mm_cache: bool = False, + ) -> list[EngineInput] | ErrorResponse: + """Core preprocessing logic for completion requests (no model/engine check). + + Called directly by render_completion_request and delegated to by + OpenAIServingCompletion.render_completion_request after its engine-aware checks. + """ + # Return error for unsupported features. + if request.suffix is not None: + return self.create_error_response("suffix is not currently supported") + + if request.echo and request.prompt_embeds is not None: + return self.create_error_response("Echo is unsupported with prompt embeds.") + + if request.prompt_logprobs is not None and request.prompt_embeds is not None: + return self.create_error_response( + "prompt_logprobs is not compatible with prompt embeds." + ) + + engine_inputs = await self.preprocess_completion( + request, + prompt_input=request.prompt, + prompt_embeds=request.prompt_embeds, + skip_mm_cache=skip_mm_cache, + ) + + return engine_inputs + + def create_error_response( + self, + message: str | Exception, + err_type: str = "BadRequestError", + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST, + param: str | None = None, + ) -> ErrorResponse: + return create_error_response(message, err_type, status_code, param) + + def validate_chat_template( + self, + request_chat_template: str | None, + chat_template_kwargs: dict[str, Any] | None, + trust_request_chat_template: bool, + ) -> ErrorResponse | None: + """Copied from OpenAIServing._validate_chat_template.""" + if not trust_request_chat_template and ( + request_chat_template is not None + or ( + chat_template_kwargs + and chat_template_kwargs.get("chat_template") is not None + ) + ): + return self.create_error_response( + "Chat template is passed with request, but " + "--trust-request-chat-template is not set. " + "Refused request with untrusted chat template." + ) + return None + + async def preprocess_completion( + self, + request: Any, + prompt_input: str | list[str] | list[int] | list[list[int]] | None, + prompt_embeds: bytes | list[bytes] | None, + *, + skip_mm_cache: bool = False, + ) -> list[EngineInput]: + """Copied from OpenAIServing._preprocess_completion.""" + prompts = list[SingletonPrompt | bytes]() + if prompt_embeds is not None: # embeds take higher priority + prompts.extend(prompt_to_seq(prompt_embeds)) + if prompt_input is not None: + prompts.extend(prompt_to_seq(prompt_input)) + return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache) + + async def preprocess_cmpl( + self, + request: Any, + prompts: Sequence[PromptType | bytes], + *, + skip_mm_cache: bool = False, + ) -> list[EngineInput]: + """Copied from OpenAIServing._preprocess_cmpl.""" + renderer = self.renderer + model_config = self.model_config + + parsed_prompts = [ + ( + prompt + if isinstance(prompt, bytes) + else parse_model_prompt(model_config, prompt) + ) + for prompt in prompts + ] + tok_params = request.build_tok_params(model_config) + + return await renderer.render_cmpl_async( + parsed_prompts, + tok_params, + prompt_extras={ + k: v + for k in ("mm_processor_kwargs", "cache_salt") + if (v := getattr(request, k, None)) is not None + }, + skip_mm_cache=skip_mm_cache, + ) + + async def preprocess_chat( + self, + request: Any, + messages: list[Any], + default_template: str | None, + default_template_content_format: ChatTemplateContentFormatOption, + default_template_kwargs: dict[str, Any] | None, + tool_dicts: list[dict[str, Any]] | None = None, + parser: type[Parser] | None = None, + *, + skip_mm_cache: bool = False, + ) -> tuple[list[ConversationMessage], list[EngineInput]]: + """Copied from OpenAIServing._preprocess_chat.""" + renderer = self.renderer + mm_config = self.model_config.multimodal_config + + default_template_kwargs = merge_kwargs( + default_template_kwargs, + dict( + tools=tool_dicts, + tokenize=( + is_mistral_tokenizer(renderer.tokenizer) + or self.model_config.enable_prompt_embeds + ), + ), + ) + + tok_params = request.build_tok_params(self.model_config) + chat_params = request.build_chat_params( + default_template, default_template_content_format + ).with_defaults( + default_template_kwargs, + default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None), + default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), + ) + + (conversation,), (engine_input,) = await renderer.render_chat_async( + [messages], + chat_params, + tok_params, + prompt_extras={ + k: v + for k in ("mm_processor_kwargs", "cache_salt") + if (v := getattr(request, k, None)) is not None + }, + skip_mm_cache=skip_mm_cache, + ) + + # tool parsing is done only if a tool_parser has been set and if + # tool_choice is not "none" (if tool_choice is "none" but a tool_parser + # is set, we want to prevent parsing a tool_call hallucinated by the LLM + # + # Exception: Mistral grammar-capable tokenizers always call + # adjust_request — even for tool_choice="none" — so that the grammar + # factory can prevent special-token leakage. + if parser is not None: + tokenizer = renderer.get_tokenizer() + tool_parser = parser.tool_parser_cls + tool_choice = getattr(request, "tool_choice", "none") + is_mistral_grammar_eligible = ( + tool_parser is not None + and is_mistral_tool_parser(tool_parser) + and is_mistral_tokenizer(tokenizer) + and tokenizer.supports_grammar + ) + should_adjust_request = ( + parser.reasoning_parser_cls is not None + or tool_choice != "none" + or is_mistral_grammar_eligible + ) + if should_adjust_request: + if not isinstance(request, ChatCompletionRequest | ResponsesRequest): + msg = ( + "Tool usage is only supported " + "for Chat Completions API or Responses API requests, " + f"but got {type(request).__name__}" + ) + raise NotImplementedError(msg) + request = parser( + tokenizer, + request.tools, + model_config=self.model_config, + chat_template_kwargs=chat_params.chat_template_kwargs, + ).adjust_request( + request=request, + ) + + return conversation, [engine_input]