[Bugfix][Tokenizer] Replace bare asserts in the DeepSeek V4 encoder (#53747)

Signed-off-by: mhuzaifa3 <mhuzaifa3@outlook.com>
This commit is contained in:
Moe Huzaifa
2026-08-25 16:30:23 +00:00
committed by GitHub
parent 3454335f7b
commit 19406fae28
2 changed files with 41 additions and 8 deletions
+24
View File
@@ -403,3 +403,27 @@ def test_deepseek_v4_matches_reference_golden_fixtures(case_id, kwargs):
expected = (FIXTURES_DIR / f"test_output_{case_id}.txt").read_text()
assert prompt == expected
def test_deepseek_v4_rejects_empty_developer_content():
with pytest.raises(ValueError):
_tokenizer().apply_chat_template(
[{"role": "developer", "content": ""}],
tokenize=False,
enable_thinking=True,
reasoning_effort="low",
)
@pytest.mark.parametrize(
"kwargs",
[
{"thinking_mode": "bogus"},
{"thinking_mode": "thinking", "reasoning_effort": "turbo"},
],
)
def test_deepseek_v4_encode_messages_rejects_invalid_arguments(kwargs):
from vllm.tokenizers.deepseek_v4_encoding import encode_messages
with pytest.raises(ValueError):
encode_messages([{"role": "user", "content": "Hello"}], **kwargs)
+17 -8
View File
@@ -225,8 +225,12 @@ def render_message(
Returns:
Encoded string for this message.
"""
assert 0 <= index < len(messages)
assert thinking_mode in ["chat", "thinking"], f"Invalid thinking_mode `{thinking_mode}`"
if not (0 <= index < len(messages)):
raise ValueError(
f"Index {index} out of range for messages list of length {len(messages)}"
)
if thinking_mode not in ["chat", "thinking"]:
raise ValueError(f"Invalid thinking_mode `{thinking_mode}`")
prompt = ""
msg = messages[index]
@@ -248,10 +252,11 @@ def render_message(
tool_calls = tool_calls_from_openai_format(tool_calls)
reasoning_effort = reasoning_effort or DEFAULT_REASONING_EFFORT
assert reasoning_effort in REASONING_EFFORT_PROMPTS, (
f"Invalid reasoning effort: {reasoning_effort}, expected one of "
f"{list(REASONING_EFFORT_PROMPTS)}"
)
if reasoning_effort not in REASONING_EFFORT_PROMPTS:
raise ValueError(
f"Invalid reasoning effort: {reasoning_effort}, expected one of "
f"{list(REASONING_EFFORT_PROMPTS)}"
)
if index == 0 and thinking_mode == "thinking":
prompt += REASONING_EFFORT_PROMPTS[reasoning_effort]
@@ -263,7 +268,8 @@ def render_message(
prompt += "\n\n" + response_format_template.format(schema=to_json(response_format))
elif role == "developer":
assert content, f"Invalid message for role `{role}`: {msg}"
if not content:
raise ValueError(f"Invalid message for role `{role}`: {msg}")
content_developer = USER_SP_TOKEN
content_developer += content
@@ -362,7 +368,10 @@ def render_message(
task = messages[index].get("task")
if task is not None:
# Task special token for internal classification tasks
assert task in VALID_TASKS, f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}"
if task not in VALID_TASKS:
raise ValueError(
f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}"
)
task_sp_token = DS_TASK_SP_TOKENS[task]
if task != "action":