[Model] Add native Dots3 NOTE multimodal support (#51255)

Signed-off-by: KurodaKanbei <mistergalahad@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
范裕达
2026-08-12 23:13:20 +08:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li Jee Jee Li
parent b1b752042f
commit 9035151d6c
28 changed files with 6468 additions and 7 deletions
@@ -127,7 +127,14 @@ def _run_verify_block():
# stub with the attribute is enough for metadata construction.
layer_name = "placeholder"
vllm_config.compilation_config.static_forward_context[layer_name] = (
types.SimpleNamespace(prefill_backend=torch.empty((1,)))
types.SimpleNamespace(
prefill_backend=torch.empty((1,)),
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
v_head_dim=V_HEAD_DIM,
)
)
init_workspace_manager(device)
+11
View File
@@ -839,6 +839,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"DeepseekOCR2ForCausalLM": _HfExamplesInfo(
"deepseek-ai/DeepSeek-OCR-2",
),
"Dots3NoteForCausalLM": _HfExamplesInfo(
"rednote-hilab/dots3.note",
trust_remote_code=True,
is_available_online=False,
),
"UnlimitedOCRForCausalLM": _HfExamplesInfo(
"baidu/Unlimited-OCR",
),
@@ -1650,6 +1655,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
trust_remote_code=True,
is_available_online=False,
),
"Dots3NoteMTPModel": _HfExamplesInfo(
"rednote-hilab/dots3.note",
speculative_model="rednote-hilab/dots3.note",
trust_remote_code=True,
is_available_online=False,
),
"Gemma4MTPModel": _HfExamplesInfo(
"google/gemma-4-E4B-it",
speculative_model="google/gemma-4-E4B-it-assistant",
+5
View File
@@ -54,6 +54,11 @@ def test_registry_imports(model_arch):
):
pytest.skip("DSparkDraftModel is only supported on CUDA and ROCm")
if model_arch in ("Dots3NoteForCausalLM", "Dots3NoteMTPModel") and not (
current_platform.is_cuda()
):
pytest.skip("Dots3 NOTE is only supported on CUDA")
# Ensure all model classes can be imported successfully
model_cls = ModelRegistry._try_load_model_cls(model_arch)
assert model_cls is not None
+275
View File
@@ -0,0 +1,275 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Iterable
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.dots_tool_parser import DotsToolParser
def _tool(name: str, properties: dict) -> ChatCompletionToolsParam:
return ChatCompletionToolsParam(
type="function",
function={
"name": name,
"description": "test tool",
"parameters": {"type": "object", "properties": properties},
},
)
def _request(tools: list[ChatCompletionToolsParam]) -> ChatCompletionRequest:
return ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
tools=tools,
tool_choice="auto",
)
def _stream(
parser: DotsToolParser,
chunks: Iterable[str],
request: ChatCompletionRequest,
) -> list[DeltaMessage]:
previous_text = ""
messages: list[DeltaMessage] = []
for chunk in chunks:
current_text = previous_text + chunk
delta = parser.extract_tool_calls_streaming(
previous_text,
current_text,
chunk,
[],
[],
[],
request,
)
if delta is not None:
messages.append(delta)
previous_text = current_text
return messages
@pytest.fixture
def parser() -> DotsToolParser:
return DotsToolParser(MagicMock())
def test_registered_in_tool_parser_manager() -> None:
assert ToolParserManager.get_tool_parser("dots") is DotsToolParser
def test_non_stream_xml_converts_schema_types_and_resolves_ref(
parser: DotsToolParser,
) -> None:
tool = ChatCompletionToolsParam(
type="function",
function={
"name": "set_location",
"description": "Set location",
"parameters": {
"type": "object",
"properties": {
"location": {"$ref": "#/$defs/Location"},
"days": {"type": "integer"},
"include_weather": {"type": "boolean"},
},
"$defs": {
"Location": {
"type": "object",
"properties": {"city": {"type": "string"}},
}
},
},
},
)
request = _request([tool])
text = (
"ok<dots_function_call>"
'<invoke name="set_location">'
'<parameter name="location">{"city": "Shanghai"}</parameter>'
'<parameter name="days">3</parameter>'
'<parameter name="include_weather">true</parameter>'
"</invoke>"
"</dots_function_call>"
)
result = parser.extract_tool_calls(text, request)
assert result.content == "ok"
assert result.tools_called
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "set_location"
assert json.loads(result.tool_calls[0].function.arguments) == {
"location": {"city": "Shanghai"},
"days": 3,
"include_weather": True,
}
def test_non_stream_supports_multiple_invokes_and_json_fallback(
parser: DotsToolParser,
) -> None:
tools = [
_tool("search", {"query": {"type": "string"}}),
_tool("open", {"id": {"type": "integer"}}),
]
request = _request(tools)
text = (
"<dots_function_call>"
'<invoke name="search"><parameter name="query">chairs</parameter></invoke>'
'<invoke name="open"><parameter name="id">7</parameter></invoke>'
"</dots_function_call>"
'<dots_function_call>{"name":"search","arguments":{"query":"tables"}}'
"</dots_function_call>"
)
result = parser.extract_tool_calls(text, request)
assert [call.function.name for call in result.tool_calls] == [
"search",
"open",
"search",
]
assert [json.loads(call.function.arguments) for call in result.tool_calls] == [
{"query": "chairs"},
{"id": 7},
{"query": "tables"},
]
def test_non_stream_unknown_tool_is_left_as_content(parser: DotsToolParser) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
text = (
"<dots_function_call>"
'<invoke name="ghost"><parameter name="query">chairs</parameter></invoke>'
"</dots_function_call>"
)
result = parser.extract_tool_calls(text, request)
assert not result.tools_called
assert result.tool_calls == []
assert result.content == text
def test_streaming_buffers_partial_marker_and_emits_all_complete_calls(
parser: DotsToolParser,
) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
chunks = [
"visible<dots_func",
(
"tion_call>"
'<invoke name="search"><parameter name="query">chairs</parameter></invoke>'
"</dots_function_call>"
"<dots_function_call>"
'<invoke name="search"><parameter name="query">tables</parameter></invoke>'
"</dots_function_call>"
),
]
messages = _stream(parser, chunks, request)
assert "".join(message.content or "" for message in messages) == "visible"
calls = [call for message in messages for call in message.tool_calls]
assert [call.index for call in calls] == [0, 1]
assert [json.loads(call.function.arguments or "") for call in calls] == [
{"query": "chairs"},
{"query": "tables"},
]
def test_streaming_filters_unknown_tools_and_surfaces_content(
parser: DotsToolParser,
) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
text = (
"<dots_function_call>"
'<invoke name="ghost"><parameter name="query">chairs</parameter></invoke>'
"</dots_function_call>"
)
messages = _stream(parser, [text], request)
assert len(messages) == 1
assert messages[0].tool_calls == []
assert "ghost" in (messages[0].content or "")
assert parser._buffer == ""
def test_streaming_malformed_block_does_not_block_later_valid_call(
parser: DotsToolParser,
) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
messages = _stream(
parser,
[
"<dots_function_call>garbage</dots_function_call>",
(
"<dots_function_call>"
'<invoke name="search">'
'<parameter name="query">chairs</parameter>'
"</invoke></dots_function_call>"
),
],
request,
)
assert messages[0].content == "garbage"
assert messages[0].tool_calls == []
assert messages[1].tool_calls[0].function.name == "search"
def test_streaming_strips_stray_end_marker(parser: DotsToolParser) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
messages = _stream(parser, ["some text </dots_function_call>"], request)
assert len(messages) == 1
assert messages[0].content == "some text "
assert messages[0].tool_calls == []
def test_streaming_flushes_partial_opening_marker_at_eof(
parser: DotsToolParser,
) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
messages = _stream(parser, ["answer <dots_func"], request)
assert messages[0].content == "answer "
assert parser.flush_pending_normal_text() == "<dots_func"
assert parser.flush_pending_normal_text() == ""
def test_streaming_emits_complete_json_before_end_marker_without_duplication(
parser: DotsToolParser,
) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
messages = _stream(
parser,
[
"<dots_function_call>",
'{"name":"search","arguments":{"query":"chairs"}}',
"</dots_function_call>",
],
request,
)
calls = [call for message in messages for call in message.tool_calls]
assert [call.function.name for call in calls] == ["search", None]
assert "".join(call.function.arguments or "" for call in calls) == (
'{"query": "chairs"}'
)
assert messages[-1].tool_calls == calls
+3
View File
@@ -78,6 +78,7 @@ def test_mla_kv_cache_spec_uses_layer_cache_dtype(
kv_cache_dtype=cache_dtype,
head_size=576,
non_causal_multi_token_decode=False,
sliding_window=None,
)
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=64), model_config=None
@@ -460,6 +461,7 @@ class MockSparseMLAAttentionLayer:
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.v_head_dim = v_head_dim
self.q_lora_rank = None
self.kv_lora_rank = kv_lora_rank
# Compute weight matrices in the format expected by forward_impl
@@ -583,6 +585,7 @@ class MockMLAAttentionLayer(MLAAttention):
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.v_head_dim = v_head_dim
self.q_lora_rank = None
self.kv_lora_rank = kv_lora_rank
# Compute weight matrices from kv_b_proj (like MLAAttention does)