From cc09352e5c50ca6b7d2c6d79b29d6d3b18df3046 Mon Sep 17 00:00:00 2001 From: adenzhou1350 Date: Fri, 18 Sep 2026 18:44:39 +0800 Subject: [PATCH] [Bugfix][Multimodal] Preserve DeepSeek V4 image block spacing (#56882) Signed-off-by: Xucheng Zhou Signed-off-by: Isotr0py Co-authored-by: OpenAI Codex Co-authored-by: Isotr0py Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../fixtures/deepseek_v4/test_input_5.json | 39 +++++++++++++++++++ .../fixtures/deepseek_v4/test_output_5.txt | 9 +++++ tests/tokenizers_/test_deepseek_v4.py | 14 +++---- vllm/renderers/deepseek_v4.py | 4 +- vllm/tokenizers/deepseek_v4_encoding.py | 6 +-- 5 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json create mode 100644 tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt diff --git a/tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json b/tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json new file mode 100644 index 0000000000..2a2fa6a89e --- /dev/null +++ b/tests/tokenizers_/fixtures/deepseek_v4/test_input_5.json @@ -0,0 +1,39 @@ +{ + "thinking_mode": "thinking", + "reasoning_effort": "max", + "messages": [ + { + "role": "system", + "content": "You are a helpful vision assistant." + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "请按“第一张、第二张”的顺序回答：第一张图" + }, + { + "type": "image_url", + "image_url": { + "url": "examples/images/carrots.jpeg" + } + }, + { + "type": "text", + "text": "和第二张图" + }, + { + "type": "image_url", + "image_url": { + "url": "examples/images/corn.jpeg" + } + }, + { + "type": "text", + "text": "中分别是什么食材？它们通常食用的部位分别是什么？" + } + ] + } + ] +} diff --git a/tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt b/tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt new file mode 100644 index 0000000000..1203dc8db1 --- /dev/null +++ b/tests/tokenizers_/fixtures/deepseek_v4/test_output_5.txt @@ -0,0 +1,9 @@ +<｜begin▁of▁sentence｜>You are a helpful vision assistant.<｜User｜>请按“第一张、第二张”的顺序回答：第一张图 + +<｜deepseek_image｜> + +和第二张图 + +<｜deepseek_image｜> + +中分别是什么食材？它们通常食用的部位分别是什么？<｜Assistant｜> \ No newline at end of file diff --git a/tests/tokenizers_/test_deepseek_v4.py b/tests/tokenizers_/test_deepseek_v4.py index f7bc5909ba..4403c2a877 100644 --- a/tests/tokenizers_/test_deepseek_v4.py +++ b/tests/tokenizers_/test_deepseek_v4.py @@ -54,13 +54,9 @@ def _load_reference_case(case_id: int): def _render_reference_case(case_id: int, **kwargs): messages, tools = _load_reference_case(case_id) - conversation, _, _ = parse_chat_messages( - messages, - _model_config(), - content_format="string", - ) + # Preserve content blocks for encoding without fetching fixture media. return _tokenizer().apply_chat_template( - conversation=conversation, + conversation=messages, messages=messages, tools=tools, tokenize=False, @@ -397,6 +393,7 @@ def test_deepseek_v4_maps_xhigh_to_high_reasoning_effort(): (2, {"thinking": True, "reasoning_effort": "low"}), (3, {"thinking": True, "reasoning_effort": "low"}), (4, {"thinking": False}), + (5, {"thinking": False}), ], ) def test_deepseek_v4_matches_reference_golden_fixtures(case_id, kwargs): @@ -447,7 +444,10 @@ def test_deepseek_v4_image_blocks_become_placeholders(): thinking=False, ) - assert "<｜User｜>first:<｜deepseek_image｜>second:<｜deepseek_image｜>" in prompt + assert ( + "<｜User｜>first:\n\n<｜deepseek_image｜>\n\n" + "second:\n\n<｜deepseek_image｜>" in prompt + ) def test_deepseek_v4_image_sentinel_ids_match_tokenizer(): diff --git a/vllm/renderers/deepseek_v4.py b/vllm/renderers/deepseek_v4.py index e93069209a..ede2e9530c 100644 --- a/vllm/renderers/deepseek_v4.py +++ b/vllm/renderers/deepseek_v4.py @@ -40,7 +40,7 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]): conversation, mm_data, mm_uuids = parse_chat_messages( messages, self.model_config, - content_format="string", + content_format="openai", media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, ) @@ -67,7 +67,7 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]): conversation, mm_data, mm_uuids = await parse_chat_messages_async( messages, self.model_config, - content_format="string", + content_format="openai", media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, ) diff --git a/vllm/tokenizers/deepseek_v4_encoding.py b/vllm/tokenizers/deepseek_v4_encoding.py index fb5bc44e85..37dc559366 100644 --- a/vllm/tokenizers/deepseek_v4_encoding.py +++ b/vllm/tokenizers/deepseek_v4_encoding.py @@ -224,7 +224,7 @@ def flatten_content_blocks(content: Any) -> Any: raise ValueError( f"Unsupported content block type: {block.get('type')!r}" ) - return "".join(parts) + return "\n\n".join(parts) def render_message( @@ -267,7 +267,7 @@ def render_message( ) role = msg.get("role") - content = msg.get("content") + content = flatten_content_blocks(msg.get("content")) tools = msg.get("tools") response_format = msg.get("response_format") tool_calls = msg.get("tool_calls") @@ -337,7 +337,7 @@ def render_message( parts.append(f"[Unsupported {block_type}]") prompt += "\n\n".join(parts) else: - prompt += flatten_content_blocks(content) or "" + prompt += content or "" elif role == "latest_reminder": prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format(content=content)