[Model] Add native Dots3 NOTE multimodal support (#51255)
Signed-off-by: KurodaKanbei <mistergalahad@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
Jee Jee Li
parent
b1b752042f
commit
9035151d6c
@@ -127,7 +127,14 @@ def _run_verify_block():
|
||||
# stub with the attribute is enough for metadata construction.
|
||||
layer_name = "placeholder"
|
||||
vllm_config.compilation_config.static_forward_context[layer_name] = (
|
||||
types.SimpleNamespace(prefill_backend=torch.empty((1,)))
|
||||
types.SimpleNamespace(
|
||||
prefill_backend=torch.empty((1,)),
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=V_HEAD_DIM,
|
||||
)
|
||||
)
|
||||
|
||||
init_workspace_manager(device)
|
||||
|
||||
@@ -839,6 +839,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"DeepseekOCR2ForCausalLM": _HfExamplesInfo(
|
||||
"deepseek-ai/DeepSeek-OCR-2",
|
||||
),
|
||||
"Dots3NoteForCausalLM": _HfExamplesInfo(
|
||||
"rednote-hilab/dots3.note",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"UnlimitedOCRForCausalLM": _HfExamplesInfo(
|
||||
"baidu/Unlimited-OCR",
|
||||
),
|
||||
@@ -1650,6 +1655,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"Dots3NoteMTPModel": _HfExamplesInfo(
|
||||
"rednote-hilab/dots3.note",
|
||||
speculative_model="rednote-hilab/dots3.note",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"Gemma4MTPModel": _HfExamplesInfo(
|
||||
"google/gemma-4-E4B-it",
|
||||
speculative_model="google/gemma-4-E4B-it-assistant",
|
||||
|
||||
@@ -54,6 +54,11 @@ def test_registry_imports(model_arch):
|
||||
):
|
||||
pytest.skip("DSparkDraftModel is only supported on CUDA and ROCm")
|
||||
|
||||
if model_arch in ("Dots3NoteForCausalLM", "Dots3NoteMTPModel") and not (
|
||||
current_platform.is_cuda()
|
||||
):
|
||||
pytest.skip("Dots3 NOTE is only supported on CUDA")
|
||||
|
||||
# Ensure all model classes can be imported successfully
|
||||
model_cls = ModelRegistry._try_load_model_cls(model_arch)
|
||||
assert model_cls is not None
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.dots_tool_parser import DotsToolParser
|
||||
|
||||
|
||||
def _tool(name: str, properties: dict) -> ChatCompletionToolsParam:
|
||||
return ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": name,
|
||||
"description": "test tool",
|
||||
"parameters": {"type": "object", "properties": properties},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _request(tools: list[ChatCompletionToolsParam]) -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
)
|
||||
|
||||
|
||||
def _stream(
|
||||
parser: DotsToolParser,
|
||||
chunks: Iterable[str],
|
||||
request: ChatCompletionRequest,
|
||||
) -> list[DeltaMessage]:
|
||||
previous_text = ""
|
||||
messages: list[DeltaMessage] = []
|
||||
for chunk in chunks:
|
||||
current_text = previous_text + chunk
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
chunk,
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
request,
|
||||
)
|
||||
if delta is not None:
|
||||
messages.append(delta)
|
||||
previous_text = current_text
|
||||
return messages
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser() -> DotsToolParser:
|
||||
return DotsToolParser(MagicMock())
|
||||
|
||||
|
||||
def test_registered_in_tool_parser_manager() -> None:
|
||||
assert ToolParserManager.get_tool_parser("dots") is DotsToolParser
|
||||
|
||||
|
||||
def test_non_stream_xml_converts_schema_types_and_resolves_ref(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
tool = ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "set_location",
|
||||
"description": "Set location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"$ref": "#/$defs/Location"},
|
||||
"days": {"type": "integer"},
|
||||
"include_weather": {"type": "boolean"},
|
||||
},
|
||||
"$defs": {
|
||||
"Location": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
request = _request([tool])
|
||||
text = (
|
||||
"ok<dots_function_call>"
|
||||
'<invoke name="set_location">'
|
||||
'<parameter name="location">{"city": "Shanghai"}</parameter>'
|
||||
'<parameter name="days">3</parameter>'
|
||||
'<parameter name="include_weather">true</parameter>'
|
||||
"</invoke>"
|
||||
"</dots_function_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(text, request)
|
||||
|
||||
assert result.content == "ok"
|
||||
assert result.tools_called
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "set_location"
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {
|
||||
"location": {"city": "Shanghai"},
|
||||
"days": 3,
|
||||
"include_weather": True,
|
||||
}
|
||||
|
||||
|
||||
def test_non_stream_supports_multiple_invokes_and_json_fallback(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
tools = [
|
||||
_tool("search", {"query": {"type": "string"}}),
|
||||
_tool("open", {"id": {"type": "integer"}}),
|
||||
]
|
||||
request = _request(tools)
|
||||
text = (
|
||||
"<dots_function_call>"
|
||||
'<invoke name="search"><parameter name="query">chairs</parameter></invoke>'
|
||||
'<invoke name="open"><parameter name="id">7</parameter></invoke>'
|
||||
"</dots_function_call>"
|
||||
'<dots_function_call>{"name":"search","arguments":{"query":"tables"}}'
|
||||
"</dots_function_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(text, request)
|
||||
|
||||
assert [call.function.name for call in result.tool_calls] == [
|
||||
"search",
|
||||
"open",
|
||||
"search",
|
||||
]
|
||||
assert [json.loads(call.function.arguments) for call in result.tool_calls] == [
|
||||
{"query": "chairs"},
|
||||
{"id": 7},
|
||||
{"query": "tables"},
|
||||
]
|
||||
|
||||
|
||||
def test_non_stream_unknown_tool_is_left_as_content(parser: DotsToolParser) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
text = (
|
||||
"<dots_function_call>"
|
||||
'<invoke name="ghost"><parameter name="query">chairs</parameter></invoke>'
|
||||
"</dots_function_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(text, request)
|
||||
|
||||
assert not result.tools_called
|
||||
assert result.tool_calls == []
|
||||
assert result.content == text
|
||||
|
||||
|
||||
def test_streaming_buffers_partial_marker_and_emits_all_complete_calls(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
chunks = [
|
||||
"visible<dots_func",
|
||||
(
|
||||
"tion_call>"
|
||||
'<invoke name="search"><parameter name="query">chairs</parameter></invoke>'
|
||||
"</dots_function_call>"
|
||||
"<dots_function_call>"
|
||||
'<invoke name="search"><parameter name="query">tables</parameter></invoke>'
|
||||
"</dots_function_call>"
|
||||
),
|
||||
]
|
||||
|
||||
messages = _stream(parser, chunks, request)
|
||||
|
||||
assert "".join(message.content or "" for message in messages) == "visible"
|
||||
calls = [call for message in messages for call in message.tool_calls]
|
||||
assert [call.index for call in calls] == [0, 1]
|
||||
assert [json.loads(call.function.arguments or "") for call in calls] == [
|
||||
{"query": "chairs"},
|
||||
{"query": "tables"},
|
||||
]
|
||||
|
||||
|
||||
def test_streaming_filters_unknown_tools_and_surfaces_content(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
text = (
|
||||
"<dots_function_call>"
|
||||
'<invoke name="ghost"><parameter name="query">chairs</parameter></invoke>'
|
||||
"</dots_function_call>"
|
||||
)
|
||||
|
||||
messages = _stream(parser, [text], request)
|
||||
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_calls == []
|
||||
assert "ghost" in (messages[0].content or "")
|
||||
assert parser._buffer == ""
|
||||
|
||||
|
||||
def test_streaming_malformed_block_does_not_block_later_valid_call(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
messages = _stream(
|
||||
parser,
|
||||
[
|
||||
"<dots_function_call>garbage</dots_function_call>",
|
||||
(
|
||||
"<dots_function_call>"
|
||||
'<invoke name="search">'
|
||||
'<parameter name="query">chairs</parameter>'
|
||||
"</invoke></dots_function_call>"
|
||||
),
|
||||
],
|
||||
request,
|
||||
)
|
||||
|
||||
assert messages[0].content == "garbage"
|
||||
assert messages[0].tool_calls == []
|
||||
assert messages[1].tool_calls[0].function.name == "search"
|
||||
|
||||
|
||||
def test_streaming_strips_stray_end_marker(parser: DotsToolParser) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
messages = _stream(parser, ["some text </dots_function_call>"], request)
|
||||
|
||||
assert len(messages) == 1
|
||||
assert messages[0].content == "some text "
|
||||
assert messages[0].tool_calls == []
|
||||
|
||||
|
||||
def test_streaming_flushes_partial_opening_marker_at_eof(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
|
||||
messages = _stream(parser, ["answer <dots_func"], request)
|
||||
|
||||
assert messages[0].content == "answer "
|
||||
assert parser.flush_pending_normal_text() == "<dots_func"
|
||||
assert parser.flush_pending_normal_text() == ""
|
||||
|
||||
|
||||
def test_streaming_emits_complete_json_before_end_marker_without_duplication(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
|
||||
messages = _stream(
|
||||
parser,
|
||||
[
|
||||
"<dots_function_call>",
|
||||
'{"name":"search","arguments":{"query":"chairs"}}',
|
||||
"</dots_function_call>",
|
||||
],
|
||||
request,
|
||||
)
|
||||
|
||||
calls = [call for message in messages for call in message.tool_calls]
|
||||
assert [call.function.name for call in calls] == ["search", None]
|
||||
assert "".join(call.function.arguments or "" for call in calls) == (
|
||||
'{"query": "chairs"}'
|
||||
)
|
||||
assert messages[-1].tool_calls == calls
|
||||
@@ -78,6 +78,7 @@ def test_mla_kv_cache_spec_uses_layer_cache_dtype(
|
||||
kv_cache_dtype=cache_dtype,
|
||||
head_size=576,
|
||||
non_causal_multi_token_decode=False,
|
||||
sliding_window=None,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=64), model_config=None
|
||||
@@ -460,6 +461,7 @@ class MockSparseMLAAttentionLayer:
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.q_lora_rank = None
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
|
||||
# Compute weight matrices in the format expected by forward_impl
|
||||
@@ -583,6 +585,7 @@ class MockMLAAttentionLayer(MLAAttention):
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.q_lora_rank = None
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
|
||||
# Compute weight matrices from kv_b_proj (like MLAAttention does)
|
||||
|
||||
Reference in New Issue
Block a user