[Feature][Frontend] Add DeepSeek-V4 FIM completion rendering (#44229)

Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com>
Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>

tests/entrypoints/openai/completion/test_completion_error.py

@@ -14,8 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.scale_out.render.serving import ServingRender
+from vllm.entrypoints.serve.engine.protocol import ErrorResponse
 from vllm.exceptions import GenerationError, VLLMValidationError
 from vllm.outputs import CompletionOutput, RequestOutput
+from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
 from vllm.renderers.hf import HfRenderer
 from vllm.renderers.online_renderer import OnlineRenderer
 from vllm.tokenizers.registry import cached_tokenizer_from_config
@@ -24,6 +26,9 @@ from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
 
 MODEL_NAME = "openai-community/gpt2"
 MODEL_NAME_SHORT = "gpt2"
+DEEPSEEK_V4_FIM_BEGIN = "<｜fim▁begin｜>"
+DEEPSEEK_V4_FIM_HOLE = "<｜fim▁hole｜>"
+DEEPSEEK_V4_FIM_END = "<｜fim▁end｜>"
 _PER_REQUEST_STATS = RequestStateStats(
     queued_ts=1.0,
     scheduled_ts=1.5,
@@ -302,6 +307,179 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
     assert response.metrics.speculative_decoding.num_spec_steps == 2
 
 
+def _build_online_renderer_for_completion(
+    model_config: MockModelConfig,
+    renderer: Any | None = None,
+) -> OnlineRenderer:
+    if renderer is None:
+        renderer = MagicMock()
+        renderer.render_completion_suffix.return_value = None
+
+    return OnlineRenderer(
+        model_config=model_config,
+        renderer=renderer,
+        request_logger=None,
+        chat_template=None,
+        chat_template_content_format="auto",
+    )
+
+
+def _deepseek_v4_suffix_renderer(
+    model_config: MockModelConfig,
+) -> DeepseekV4Renderer:
+    return DeepseekV4Renderer(
+        MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
+        None,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip_global_cleanup
+async def test_completion_suffix_uses_renderer_capability():
+    model_config = MockModelConfig()
+    online_renderer = _build_online_renderer_for_completion(
+        model_config,
+        renderer=_deepseek_v4_suffix_renderer(model_config),
+    )
+    online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}])
+
+    request = CompletionRequest(
+        model=MODEL_NAME,
+        prompt="def fib(n):\n    return ",
+        suffix="\n\nprint(fib(10))",
+        max_tokens=64,
+    )
+
+    result = await online_renderer.render_completion(request)
+
+    assert result == [{"ok": True}]
+    online_renderer.preprocess_completion.assert_awaited_once()
+    assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == (
+        f"{DEEPSEEK_V4_FIM_BEGIN}def fib(n):\n    return "
+        f"{DEEPSEEK_V4_FIM_HOLE}\n\nprint(fib(10))"
+        f"{DEEPSEEK_V4_FIM_END}"
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip_global_cleanup
+async def test_completion_suffix_supports_text_prompt_list():
+    model_config = MockModelConfig()
+    online_renderer = _build_online_renderer_for_completion(
+        model_config,
+        renderer=_deepseek_v4_suffix_renderer(model_config),
+    )
+    online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}])
+
+    request = CompletionRequest(
+        model=MODEL_NAME,
+        prompt=["prefix A", "prefix B"],
+        suffix=" suffix",
+    )
+
+    result = await online_renderer.render_completion(request)
+
+    assert result == [{"ok": True}]
+    assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == [
+        f"{DEEPSEEK_V4_FIM_BEGIN}prefix A"
+        f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
+        f"{DEEPSEEK_V4_FIM_BEGIN}prefix B"
+        f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip_global_cleanup
+async def test_completion_suffix_rejects_renderer_without_fim_support():
+    model_config = MockModelConfig()
+    online_renderer = _build_online_renderer_for_completion(model_config)
+    online_renderer.preprocess_completion = AsyncMock()
+
+    request = CompletionRequest(
+        model=MODEL_NAME,
+        prompt="prefix",
+        suffix="suffix",
+    )
+
+    result = await online_renderer.render_completion(request)
+
+    assert isinstance(result, ErrorResponse)
+    assert "FIM completion rendering" in result.error.message
+    online_renderer.preprocess_completion.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip_global_cleanup
+async def test_completion_suffix_rejects_echo():
+    model_config = MockModelConfig()
+    online_renderer = _build_online_renderer_for_completion(
+        model_config,
+        renderer=_deepseek_v4_suffix_renderer(model_config),
+    )
+    online_renderer.preprocess_completion = AsyncMock()
+
+    request = CompletionRequest(
+        model=MODEL_NAME,
+        prompt="prefix",
+        suffix="suffix",
+        echo=True,
+    )
+
+    result = await online_renderer.render_completion(request)
+
+    assert isinstance(result, ErrorResponse)
+    assert "Echo is unsupported with suffix" in result.error.message
+    online_renderer.preprocess_completion.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip_global_cleanup
+async def test_completion_suffix_rejects_prompt_embeds():
+    model_config = MockModelConfig()
+    online_renderer = _build_online_renderer_for_completion(
+        model_config,
+        renderer=_deepseek_v4_suffix_renderer(model_config),
+    )
+    online_renderer.preprocess_completion = AsyncMock()
+
+    request = CompletionRequest(
+        model=MODEL_NAME,
+        prompt="prefix",
+        suffix="suffix",
+        prompt_embeds=b"embeds",
+    )
+
+    result = await online_renderer.render_completion(request)
+
+    assert isinstance(result, ErrorResponse)
+    assert "suffix is not supported with prompt_embeds" in result.error.message
+    online_renderer.preprocess_completion.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip_global_cleanup
+@pytest.mark.parametrize("prompt", ([1, 2, 3], [[1, 2, 3]]))
+async def test_completion_suffix_rejects_token_prompt(prompt):
+    model_config = MockModelConfig()
+    online_renderer = _build_online_renderer_for_completion(
+        model_config,
+        renderer=_deepseek_v4_suffix_renderer(model_config),
+    )
+    online_renderer.preprocess_completion = AsyncMock()
+
+    request = CompletionRequest(
+        model=MODEL_NAME,
+        prompt=prompt,
+        suffix="suffix",
+    )
+
+    result = await online_renderer.render_completion(request)
+
+    assert isinstance(result, ErrorResponse)
+    assert "requires text prompt input" in result.error.message
+    online_renderer.preprocess_completion.assert_not_awaited()
+
+
 @pytest.mark.asyncio
 async def test_completion_error_non_stream():
     """Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""

vllm/entrypoints/openai/completion/serving.py

@@ -116,9 +116,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
         See https://platform.openai.com/docs/api-reference/completions/create
         for the API specification. This API mimics the OpenAI Completion API.
 
-        NOTE: Currently we do not support the following feature:
-            - suffix (the language models we currently support do not support
-            suffix)
+        NOTE: suffix is only supported by models that implement FIM rendering.
         """
         return await self._with_kv_transfer_rejection_cleanup(
             self._create_completion(request, raw_request), request, raw_request

vllm/renderers/base.py

@@ -193,6 +193,10 @@ class BaseRenderer(ABC, Generic[_T]):
 
         return tokenizer
 
+    def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
+        """Render OpenAI completion suffix input when the renderer supports FIM."""
+        return None
+
     def _decode(self, *args, **kwargs):
         return self.get_tokenizer().decode(*args, **kwargs)
 

vllm/renderers/deepseek_v4.py

@@ -16,6 +16,10 @@ from .inputs import DictPrompt
 from .inputs.preprocess import parse_dec_only_prompt
 from .params import ChatParams
 
+_FIM_BEGIN = "<｜fim▁begin｜>"
+_FIM_HOLE = "<｜fim▁hole｜>"
+_FIM_END = "<｜fim▁end｜>"
+
 
 class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
     def __init__(
@@ -32,6 +36,9 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
     def _apply_chat_template(self, *args, **kwargs):
         return self.get_tokenizer().apply_chat_template(*args, **kwargs)
 
+    def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
+        return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}"
+
     def render_messages(
         self,
         messages: list[ChatCompletionMessageParam],

vllm/renderers/online_renderer.py

@@ -4,7 +4,7 @@ import time
 from collections.abc import Sequence
 from dataclasses import dataclass
 from http import HTTPStatus
-from typing import TYPE_CHECKING, Any, TypeAlias
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
 
 from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
 from openai.types.responses.tool import Mcp, Tool
@@ -548,9 +548,58 @@ class OnlineRenderer:
         Called directly by render_completion_request and delegated to by
         OpenAIServingCompletion.render_completion_request after its engine-aware checks.
         """
-        # Return error for unsupported features.
+        prompt_input = request.prompt
         if request.suffix is not None:
-            return self.create_error_response("suffix is not currently supported")
+            if request.echo:
+                return self.create_error_response(
+                    "Echo is unsupported with suffix.",
+                    param="suffix",
+                )
+
+            if request.prompt_embeds is not None:
+                return self.create_error_response(
+                    "suffix is not supported with prompt_embeds",
+                    param="suffix",
+                )
+
+            if request.truncate_prompt_tokens is not None:
+                return self.create_error_response(
+                    "suffix is not supported with truncate_prompt_tokens",
+                    param="suffix",
+                )
+
+            if isinstance(request.prompt, str):
+                rendered_prompt = self.renderer.render_completion_suffix(
+                    request.prompt, request.suffix
+                )
+                if rendered_prompt is None:
+                    return self.create_error_response(
+                        "suffix is only supported for models with FIM completion "
+                        "rendering",
+                        param="suffix",
+                    )
+                prompt_input = rendered_prompt
+            elif isinstance(request.prompt, list) and all(
+                isinstance(prompt, str) for prompt in request.prompt
+            ):
+                rendered_prompts = []
+                for prompt in cast(list[str], request.prompt):
+                    rendered_prompt = self.renderer.render_completion_suffix(
+                        prompt, request.suffix
+                    )
+                    if rendered_prompt is None:
+                        return self.create_error_response(
+                            "suffix is only supported for models with FIM completion "
+                            "rendering",
+                            param="suffix",
+                        )
+                    rendered_prompts.append(rendered_prompt)
+                prompt_input = rendered_prompts
+            else:
+                return self.create_error_response(
+                    "suffix requires text prompt input for FIM completion rendering",
+                    param="suffix",
+                )
 
         if request.echo and request.prompt_embeds is not None:
             return self.create_error_response("Echo is unsupported with prompt embeds.")
@@ -562,7 +611,7 @@ class OnlineRenderer:
 
         engine_inputs = await self.preprocess_completion(
             request,
-            prompt_input=request.prompt,
+            prompt_input=prompt_input,
             prompt_embeds=request.prompt_embeds,
             skip_mm_cache=skip_mm_cache,
         )