[Feature][Frontend] Add DeepSeek-V4 FIM completion rendering (#44229)
Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
QwertyJack
Chauncey
parent
157bcb7c48
commit
d95a896ea6
@@ -14,8 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
|||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
|
from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
|
||||||
from vllm.renderers.hf import HfRenderer
|
from vllm.renderers.hf import HfRenderer
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||||
@@ -24,6 +26,9 @@ from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
|
|||||||
|
|
||||||
MODEL_NAME = "openai-community/gpt2"
|
MODEL_NAME = "openai-community/gpt2"
|
||||||
MODEL_NAME_SHORT = "gpt2"
|
MODEL_NAME_SHORT = "gpt2"
|
||||||
|
DEEPSEEK_V4_FIM_BEGIN = "<|fim▁begin|>"
|
||||||
|
DEEPSEEK_V4_FIM_HOLE = "<|fim▁hole|>"
|
||||||
|
DEEPSEEK_V4_FIM_END = "<|fim▁end|>"
|
||||||
_PER_REQUEST_STATS = RequestStateStats(
|
_PER_REQUEST_STATS = RequestStateStats(
|
||||||
queued_ts=1.0,
|
queued_ts=1.0,
|
||||||
scheduled_ts=1.5,
|
scheduled_ts=1.5,
|
||||||
@@ -302,6 +307,179 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
|
|||||||
assert response.metrics.speculative_decoding.num_spec_steps == 2
|
assert response.metrics.speculative_decoding.num_spec_steps == 2
|
||||||
|
|
||||||
|
|
||||||
|
def _build_online_renderer_for_completion(
|
||||||
|
model_config: MockModelConfig,
|
||||||
|
renderer: Any | None = None,
|
||||||
|
) -> OnlineRenderer:
|
||||||
|
if renderer is None:
|
||||||
|
renderer = MagicMock()
|
||||||
|
renderer.render_completion_suffix.return_value = None
|
||||||
|
|
||||||
|
return OnlineRenderer(
|
||||||
|
model_config=model_config,
|
||||||
|
renderer=renderer,
|
||||||
|
request_logger=None,
|
||||||
|
chat_template=None,
|
||||||
|
chat_template_content_format="auto",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _deepseek_v4_suffix_renderer(
|
||||||
|
model_config: MockModelConfig,
|
||||||
|
) -> DeepseekV4Renderer:
|
||||||
|
return DeepseekV4Renderer(
|
||||||
|
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
async def test_completion_suffix_uses_renderer_capability():
|
||||||
|
model_config = MockModelConfig()
|
||||||
|
online_renderer = _build_online_renderer_for_completion(
|
||||||
|
model_config,
|
||||||
|
renderer=_deepseek_v4_suffix_renderer(model_config),
|
||||||
|
)
|
||||||
|
online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}])
|
||||||
|
|
||||||
|
request = CompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
prompt="def fib(n):\n return ",
|
||||||
|
suffix="\n\nprint(fib(10))",
|
||||||
|
max_tokens=64,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await online_renderer.render_completion(request)
|
||||||
|
|
||||||
|
assert result == [{"ok": True}]
|
||||||
|
online_renderer.preprocess_completion.assert_awaited_once()
|
||||||
|
assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == (
|
||||||
|
f"{DEEPSEEK_V4_FIM_BEGIN}def fib(n):\n return "
|
||||||
|
f"{DEEPSEEK_V4_FIM_HOLE}\n\nprint(fib(10))"
|
||||||
|
f"{DEEPSEEK_V4_FIM_END}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
async def test_completion_suffix_supports_text_prompt_list():
|
||||||
|
model_config = MockModelConfig()
|
||||||
|
online_renderer = _build_online_renderer_for_completion(
|
||||||
|
model_config,
|
||||||
|
renderer=_deepseek_v4_suffix_renderer(model_config),
|
||||||
|
)
|
||||||
|
online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}])
|
||||||
|
|
||||||
|
request = CompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
prompt=["prefix A", "prefix B"],
|
||||||
|
suffix=" suffix",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await online_renderer.render_completion(request)
|
||||||
|
|
||||||
|
assert result == [{"ok": True}]
|
||||||
|
assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == [
|
||||||
|
f"{DEEPSEEK_V4_FIM_BEGIN}prefix A"
|
||||||
|
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
|
||||||
|
f"{DEEPSEEK_V4_FIM_BEGIN}prefix B"
|
||||||
|
f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
async def test_completion_suffix_rejects_renderer_without_fim_support():
|
||||||
|
model_config = MockModelConfig()
|
||||||
|
online_renderer = _build_online_renderer_for_completion(model_config)
|
||||||
|
online_renderer.preprocess_completion = AsyncMock()
|
||||||
|
|
||||||
|
request = CompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
prompt="prefix",
|
||||||
|
suffix="suffix",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await online_renderer.render_completion(request)
|
||||||
|
|
||||||
|
assert isinstance(result, ErrorResponse)
|
||||||
|
assert "FIM completion rendering" in result.error.message
|
||||||
|
online_renderer.preprocess_completion.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
async def test_completion_suffix_rejects_echo():
|
||||||
|
model_config = MockModelConfig()
|
||||||
|
online_renderer = _build_online_renderer_for_completion(
|
||||||
|
model_config,
|
||||||
|
renderer=_deepseek_v4_suffix_renderer(model_config),
|
||||||
|
)
|
||||||
|
online_renderer.preprocess_completion = AsyncMock()
|
||||||
|
|
||||||
|
request = CompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
prompt="prefix",
|
||||||
|
suffix="suffix",
|
||||||
|
echo=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await online_renderer.render_completion(request)
|
||||||
|
|
||||||
|
assert isinstance(result, ErrorResponse)
|
||||||
|
assert "Echo is unsupported with suffix" in result.error.message
|
||||||
|
online_renderer.preprocess_completion.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
async def test_completion_suffix_rejects_prompt_embeds():
|
||||||
|
model_config = MockModelConfig()
|
||||||
|
online_renderer = _build_online_renderer_for_completion(
|
||||||
|
model_config,
|
||||||
|
renderer=_deepseek_v4_suffix_renderer(model_config),
|
||||||
|
)
|
||||||
|
online_renderer.preprocess_completion = AsyncMock()
|
||||||
|
|
||||||
|
request = CompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
prompt="prefix",
|
||||||
|
suffix="suffix",
|
||||||
|
prompt_embeds=b"embeds",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await online_renderer.render_completion(request)
|
||||||
|
|
||||||
|
assert isinstance(result, ErrorResponse)
|
||||||
|
assert "suffix is not supported with prompt_embeds" in result.error.message
|
||||||
|
online_renderer.preprocess_completion.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
@pytest.mark.parametrize("prompt", ([1, 2, 3], [[1, 2, 3]]))
|
||||||
|
async def test_completion_suffix_rejects_token_prompt(prompt):
|
||||||
|
model_config = MockModelConfig()
|
||||||
|
online_renderer = _build_online_renderer_for_completion(
|
||||||
|
model_config,
|
||||||
|
renderer=_deepseek_v4_suffix_renderer(model_config),
|
||||||
|
)
|
||||||
|
online_renderer.preprocess_completion = AsyncMock()
|
||||||
|
|
||||||
|
request = CompletionRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
prompt=prompt,
|
||||||
|
suffix="suffix",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await online_renderer.render_completion(request)
|
||||||
|
|
||||||
|
assert isinstance(result, ErrorResponse)
|
||||||
|
assert "requires text prompt input" in result.error.message
|
||||||
|
online_renderer.preprocess_completion.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_completion_error_non_stream():
|
async def test_completion_error_non_stream():
|
||||||
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
|
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
|
||||||
|
|||||||
@@ -116,9 +116,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
|||||||
See https://platform.openai.com/docs/api-reference/completions/create
|
See https://platform.openai.com/docs/api-reference/completions/create
|
||||||
for the API specification. This API mimics the OpenAI Completion API.
|
for the API specification. This API mimics the OpenAI Completion API.
|
||||||
|
|
||||||
NOTE: Currently we do not support the following feature:
|
NOTE: suffix is only supported by models that implement FIM rendering.
|
||||||
- suffix (the language models we currently support do not support
|
|
||||||
suffix)
|
|
||||||
"""
|
"""
|
||||||
return await self._with_kv_transfer_rejection_cleanup(
|
return await self._with_kv_transfer_rejection_cleanup(
|
||||||
self._create_completion(request, raw_request), request, raw_request
|
self._create_completion(request, raw_request), request, raw_request
|
||||||
|
|||||||
@@ -193,6 +193,10 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
|
|
||||||
return tokenizer
|
return tokenizer
|
||||||
|
|
||||||
|
def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
|
||||||
|
"""Render OpenAI completion suffix input when the renderer supports FIM."""
|
||||||
|
return None
|
||||||
|
|
||||||
def _decode(self, *args, **kwargs):
|
def _decode(self, *args, **kwargs):
|
||||||
return self.get_tokenizer().decode(*args, **kwargs)
|
return self.get_tokenizer().decode(*args, **kwargs)
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,10 @@ from .inputs import DictPrompt
|
|||||||
from .inputs.preprocess import parse_dec_only_prompt
|
from .inputs.preprocess import parse_dec_only_prompt
|
||||||
from .params import ChatParams
|
from .params import ChatParams
|
||||||
|
|
||||||
|
_FIM_BEGIN = "<|fim▁begin|>"
|
||||||
|
_FIM_HOLE = "<|fim▁hole|>"
|
||||||
|
_FIM_END = "<|fim▁end|>"
|
||||||
|
|
||||||
|
|
||||||
class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
|
class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -32,6 +36,9 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
|
|||||||
def _apply_chat_template(self, *args, **kwargs):
|
def _apply_chat_template(self, *args, **kwargs):
|
||||||
return self.get_tokenizer().apply_chat_template(*args, **kwargs)
|
return self.get_tokenizer().apply_chat_template(*args, **kwargs)
|
||||||
|
|
||||||
|
def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
|
||||||
|
return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}"
|
||||||
|
|
||||||
def render_messages(
|
def render_messages(
|
||||||
self,
|
self,
|
||||||
messages: list[ChatCompletionMessageParam],
|
messages: list[ChatCompletionMessageParam],
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import time
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
from typing import TYPE_CHECKING, Any, TypeAlias
|
from typing import TYPE_CHECKING, Any, TypeAlias, cast
|
||||||
|
|
||||||
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
|
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
|
||||||
from openai.types.responses.tool import Mcp, Tool
|
from openai.types.responses.tool import Mcp, Tool
|
||||||
@@ -548,9 +548,58 @@ class OnlineRenderer:
|
|||||||
Called directly by render_completion_request and delegated to by
|
Called directly by render_completion_request and delegated to by
|
||||||
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
|
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
|
||||||
"""
|
"""
|
||||||
# Return error for unsupported features.
|
prompt_input = request.prompt
|
||||||
if request.suffix is not None:
|
if request.suffix is not None:
|
||||||
return self.create_error_response("suffix is not currently supported")
|
if request.echo:
|
||||||
|
return self.create_error_response(
|
||||||
|
"Echo is unsupported with suffix.",
|
||||||
|
param="suffix",
|
||||||
|
)
|
||||||
|
|
||||||
|
if request.prompt_embeds is not None:
|
||||||
|
return self.create_error_response(
|
||||||
|
"suffix is not supported with prompt_embeds",
|
||||||
|
param="suffix",
|
||||||
|
)
|
||||||
|
|
||||||
|
if request.truncate_prompt_tokens is not None:
|
||||||
|
return self.create_error_response(
|
||||||
|
"suffix is not supported with truncate_prompt_tokens",
|
||||||
|
param="suffix",
|
||||||
|
)
|
||||||
|
|
||||||
|
if isinstance(request.prompt, str):
|
||||||
|
rendered_prompt = self.renderer.render_completion_suffix(
|
||||||
|
request.prompt, request.suffix
|
||||||
|
)
|
||||||
|
if rendered_prompt is None:
|
||||||
|
return self.create_error_response(
|
||||||
|
"suffix is only supported for models with FIM completion "
|
||||||
|
"rendering",
|
||||||
|
param="suffix",
|
||||||
|
)
|
||||||
|
prompt_input = rendered_prompt
|
||||||
|
elif isinstance(request.prompt, list) and all(
|
||||||
|
isinstance(prompt, str) for prompt in request.prompt
|
||||||
|
):
|
||||||
|
rendered_prompts = []
|
||||||
|
for prompt in cast(list[str], request.prompt):
|
||||||
|
rendered_prompt = self.renderer.render_completion_suffix(
|
||||||
|
prompt, request.suffix
|
||||||
|
)
|
||||||
|
if rendered_prompt is None:
|
||||||
|
return self.create_error_response(
|
||||||
|
"suffix is only supported for models with FIM completion "
|
||||||
|
"rendering",
|
||||||
|
param="suffix",
|
||||||
|
)
|
||||||
|
rendered_prompts.append(rendered_prompt)
|
||||||
|
prompt_input = rendered_prompts
|
||||||
|
else:
|
||||||
|
return self.create_error_response(
|
||||||
|
"suffix requires text prompt input for FIM completion rendering",
|
||||||
|
param="suffix",
|
||||||
|
)
|
||||||
|
|
||||||
if request.echo and request.prompt_embeds is not None:
|
if request.echo and request.prompt_embeds is not None:
|
||||||
return self.create_error_response("Echo is unsupported with prompt embeds.")
|
return self.create_error_response("Echo is unsupported with prompt embeds.")
|
||||||
@@ -562,7 +611,7 @@ class OnlineRenderer:
|
|||||||
|
|
||||||
engine_inputs = await self.preprocess_completion(
|
engine_inputs = await self.preprocess_completion(
|
||||||
request,
|
request,
|
||||||
prompt_input=request.prompt,
|
prompt_input=prompt_input,
|
||||||
prompt_embeds=request.prompt_embeds,
|
prompt_embeds=request.prompt_embeds,
|
||||||
skip_mm_cache=skip_mm_cache,
|
skip_mm_cache=skip_mm_cache,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user