From d95a896ea612a072ba986a708d7efb4de1547019 Mon Sep 17 00:00:00 2001 From: jack Date: Wed, 23 Sep 2026 21:02:00 +0800 Subject: [PATCH] [Feature][Frontend] Add DeepSeek-V4 FIM completion rendering (#44229) Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: Chauncey --- .../completion/test_completion_error.py | 178 ++++++++++++++++++ vllm/entrypoints/openai/completion/serving.py | 4 +- vllm/renderers/base.py | 4 + vllm/renderers/deepseek_v4.py | 7 + vllm/renderers/online_renderer.py | 57 +++++- 5 files changed, 243 insertions(+), 7 deletions(-) diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index 28a499e129..060ae01a1d 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -14,8 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.scale_out.render.serving import ServingRender +from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.exceptions import GenerationError, VLLMValidationError from vllm.outputs import CompletionOutput, RequestOutput +from vllm.renderers.deepseek_v4 import DeepseekV4Renderer from vllm.renderers.hf import HfRenderer from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config @@ -24,6 +26,9 @@ from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats MODEL_NAME = "openai-community/gpt2" MODEL_NAME_SHORT = "gpt2" +DEEPSEEK_V4_FIM_BEGIN = "<｜fim▁begin｜>" +DEEPSEEK_V4_FIM_HOLE = "<｜fim▁hole｜>" +DEEPSEEK_V4_FIM_END = "<｜fim▁end｜>" _PER_REQUEST_STATS = RequestStateStats( queued_ts=1.0, scheduled_ts=1.5, @@ -302,6 +307,179 @@ def test_completion_metrics_carries_both_timing_and_spec_decode(): assert response.metrics.speculative_decoding.num_spec_steps == 2 +def _build_online_renderer_for_completion( + model_config: MockModelConfig, + renderer: Any | None = None, +) -> OnlineRenderer: + if renderer is None: + renderer = MagicMock() + renderer.render_completion_suffix.return_value = None + + return OnlineRenderer( + model_config=model_config, + renderer=renderer, + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + + +def _deepseek_v4_suffix_renderer( + model_config: MockModelConfig, +) -> DeepseekV4Renderer: + return DeepseekV4Renderer( + MockVllmConfig(model_config, parallel_config=MockParallelConfig()), + None, + ) + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_uses_renderer_capability(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}]) + + request = CompletionRequest( + model=MODEL_NAME, + prompt="def fib(n):\n return ", + suffix="\n\nprint(fib(10))", + max_tokens=64, + ) + + result = await online_renderer.render_completion(request) + + assert result == [{"ok": True}] + online_renderer.preprocess_completion.assert_awaited_once() + assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == ( + f"{DEEPSEEK_V4_FIM_BEGIN}def fib(n):\n return " + f"{DEEPSEEK_V4_FIM_HOLE}\n\nprint(fib(10))" + f"{DEEPSEEK_V4_FIM_END}" + ) + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_supports_text_prompt_list(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}]) + + request = CompletionRequest( + model=MODEL_NAME, + prompt=["prefix A", "prefix B"], + suffix=" suffix", + ) + + result = await online_renderer.render_completion(request) + + assert result == [{"ok": True}] + assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == [ + f"{DEEPSEEK_V4_FIM_BEGIN}prefix A" + f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}", + f"{DEEPSEEK_V4_FIM_BEGIN}prefix B" + f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}", + ] + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_rejects_renderer_without_fim_support(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion(model_config) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt="prefix", + suffix="suffix", + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "FIM completion rendering" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_rejects_echo(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt="prefix", + suffix="suffix", + echo=True, + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "Echo is unsupported with suffix" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_completion_suffix_rejects_prompt_embeds(): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt="prefix", + suffix="suffix", + prompt_embeds=b"embeds", + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "suffix is not supported with prompt_embeds" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize("prompt", ([1, 2, 3], [[1, 2, 3]])) +async def test_completion_suffix_rejects_token_prompt(prompt): + model_config = MockModelConfig() + online_renderer = _build_online_renderer_for_completion( + model_config, + renderer=_deepseek_v4_suffix_renderer(model_config), + ) + online_renderer.preprocess_completion = AsyncMock() + + request = CompletionRequest( + model=MODEL_NAME, + prompt=prompt, + suffix="suffix", + ) + + result = await online_renderer.render_completion(request) + + assert isinstance(result, ErrorResponse) + assert "requires text prompt input" in result.error.message + online_renderer.preprocess_completion.assert_not_awaited() + + @pytest.mark.asyncio async def test_completion_error_non_stream(): """Test finish_reason='error' returns 500 InternalServerError (non-streaming).""" diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index 4d892d89d5..d7e235da1b 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -116,9 +116,7 @@ class OpenAIServingCompletion(GenerateBaseServing): See https://platform.openai.com/docs/api-reference/completions/create for the API specification. This API mimics the OpenAI Completion API. - NOTE: Currently we do not support the following feature: - - suffix (the language models we currently support do not support - suffix) + NOTE: suffix is only supported by models that implement FIM rendering. """ return await self._with_kv_transfer_rejection_cleanup( self._create_completion(request, raw_request), request, raw_request diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 53e3f50b4e..87d11b7b92 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -193,6 +193,10 @@ class BaseRenderer(ABC, Generic[_T]): return tokenizer + def render_completion_suffix(self, prompt: str, suffix: str) -> str | None: + """Render OpenAI completion suffix input when the renderer supports FIM.""" + return None + def _decode(self, *args, **kwargs): return self.get_tokenizer().decode(*args, **kwargs) diff --git a/vllm/renderers/deepseek_v4.py b/vllm/renderers/deepseek_v4.py index ede2e9530c..470e21f1eb 100644 --- a/vllm/renderers/deepseek_v4.py +++ b/vllm/renderers/deepseek_v4.py @@ -16,6 +16,10 @@ from .inputs import DictPrompt from .inputs.preprocess import parse_dec_only_prompt from .params import ChatParams +_FIM_BEGIN = "<｜fim▁begin｜>" +_FIM_HOLE = "<｜fim▁hole｜>" +_FIM_END = "<｜fim▁end｜>" + class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]): def __init__( @@ -32,6 +36,9 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]): def _apply_chat_template(self, *args, **kwargs): return self.get_tokenizer().apply_chat_template(*args, **kwargs) + def render_completion_suffix(self, prompt: str, suffix: str) -> str | None: + return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}" + def render_messages( self, messages: list[ChatCompletionMessageParam], diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index f4d21dedad..e5169d70ba 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -4,7 +4,7 @@ import time from collections.abc import Sequence from dataclasses import dataclass from http import HTTPStatus -from typing import TYPE_CHECKING, Any, TypeAlias +from typing import TYPE_CHECKING, Any, TypeAlias, cast from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem from openai.types.responses.tool import Mcp, Tool @@ -548,9 +548,58 @@ class OnlineRenderer: Called directly by render_completion_request and delegated to by OpenAIServingCompletion.render_completion_request after its engine-aware checks. """ - # Return error for unsupported features. + prompt_input = request.prompt if request.suffix is not None: - return self.create_error_response("suffix is not currently supported") + if request.echo: + return self.create_error_response( + "Echo is unsupported with suffix.", + param="suffix", + ) + + if request.prompt_embeds is not None: + return self.create_error_response( + "suffix is not supported with prompt_embeds", + param="suffix", + ) + + if request.truncate_prompt_tokens is not None: + return self.create_error_response( + "suffix is not supported with truncate_prompt_tokens", + param="suffix", + ) + + if isinstance(request.prompt, str): + rendered_prompt = self.renderer.render_completion_suffix( + request.prompt, request.suffix + ) + if rendered_prompt is None: + return self.create_error_response( + "suffix is only supported for models with FIM completion " + "rendering", + param="suffix", + ) + prompt_input = rendered_prompt + elif isinstance(request.prompt, list) and all( + isinstance(prompt, str) for prompt in request.prompt + ): + rendered_prompts = [] + for prompt in cast(list[str], request.prompt): + rendered_prompt = self.renderer.render_completion_suffix( + prompt, request.suffix + ) + if rendered_prompt is None: + return self.create_error_response( + "suffix is only supported for models with FIM completion " + "rendering", + param="suffix", + ) + rendered_prompts.append(rendered_prompt) + prompt_input = rendered_prompts + else: + return self.create_error_response( + "suffix requires text prompt input for FIM completion rendering", + param="suffix", + ) if request.echo and request.prompt_embeds is not None: return self.create_error_response("Echo is unsupported with prompt embeds.") @@ -562,7 +611,7 @@ class OnlineRenderer: engine_inputs = await self.preprocess_completion( request, - prompt_input=request.prompt, + prompt_input=prompt_input, prompt_embeds=request.prompt_embeds, skip_mm_cache=skip_mm_cache, )