[Feature][Frontend] Add DeepSeek-V4 FIM completion rendering (#44229)
Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
QwertyJack
Chauncey
parent
157bcb7c48
commit
d95a896ea6
@@ -193,6 +193,10 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
|
||||
return tokenizer
|
||||
|
||||
def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
|
||||
"""Render OpenAI completion suffix input when the renderer supports FIM."""
|
||||
return None
|
||||
|
||||
def _decode(self, *args, **kwargs):
|
||||
return self.get_tokenizer().decode(*args, **kwargs)
|
||||
|
||||
|
||||
@@ -16,6 +16,10 @@ from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
|
||||
_FIM_BEGIN = "<|fim▁begin|>"
|
||||
_FIM_HOLE = "<|fim▁hole|>"
|
||||
_FIM_END = "<|fim▁end|>"
|
||||
|
||||
|
||||
class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
|
||||
def __init__(
|
||||
@@ -32,6 +36,9 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
|
||||
def _apply_chat_template(self, *args, **kwargs):
|
||||
return self.get_tokenizer().apply_chat_template(*args, **kwargs)
|
||||
|
||||
def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
|
||||
return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}"
|
||||
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
|
||||
@@ -4,7 +4,7 @@ import time
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias, cast
|
||||
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
|
||||
from openai.types.responses.tool import Mcp, Tool
|
||||
@@ -548,9 +548,58 @@ class OnlineRenderer:
|
||||
Called directly by render_completion_request and delegated to by
|
||||
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
|
||||
"""
|
||||
# Return error for unsupported features.
|
||||
prompt_input = request.prompt
|
||||
if request.suffix is not None:
|
||||
return self.create_error_response("suffix is not currently supported")
|
||||
if request.echo:
|
||||
return self.create_error_response(
|
||||
"Echo is unsupported with suffix.",
|
||||
param="suffix",
|
||||
)
|
||||
|
||||
if request.prompt_embeds is not None:
|
||||
return self.create_error_response(
|
||||
"suffix is not supported with prompt_embeds",
|
||||
param="suffix",
|
||||
)
|
||||
|
||||
if request.truncate_prompt_tokens is not None:
|
||||
return self.create_error_response(
|
||||
"suffix is not supported with truncate_prompt_tokens",
|
||||
param="suffix",
|
||||
)
|
||||
|
||||
if isinstance(request.prompt, str):
|
||||
rendered_prompt = self.renderer.render_completion_suffix(
|
||||
request.prompt, request.suffix
|
||||
)
|
||||
if rendered_prompt is None:
|
||||
return self.create_error_response(
|
||||
"suffix is only supported for models with FIM completion "
|
||||
"rendering",
|
||||
param="suffix",
|
||||
)
|
||||
prompt_input = rendered_prompt
|
||||
elif isinstance(request.prompt, list) and all(
|
||||
isinstance(prompt, str) for prompt in request.prompt
|
||||
):
|
||||
rendered_prompts = []
|
||||
for prompt in cast(list[str], request.prompt):
|
||||
rendered_prompt = self.renderer.render_completion_suffix(
|
||||
prompt, request.suffix
|
||||
)
|
||||
if rendered_prompt is None:
|
||||
return self.create_error_response(
|
||||
"suffix is only supported for models with FIM completion "
|
||||
"rendering",
|
||||
param="suffix",
|
||||
)
|
||||
rendered_prompts.append(rendered_prompt)
|
||||
prompt_input = rendered_prompts
|
||||
else:
|
||||
return self.create_error_response(
|
||||
"suffix requires text prompt input for FIM completion rendering",
|
||||
param="suffix",
|
||||
)
|
||||
|
||||
if request.echo and request.prompt_embeds is not None:
|
||||
return self.create_error_response("Echo is unsupported with prompt embeds.")
|
||||
@@ -562,7 +611,7 @@ class OnlineRenderer:
|
||||
|
||||
engine_inputs = await self.preprocess_completion(
|
||||
request,
|
||||
prompt_input=request.prompt,
|
||||
prompt_input=prompt_input,
|
||||
prompt_embeds=request.prompt_embeds,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user