[Frontend] Add Streaming Parser Engine and new Qwen3 Parser (#45413)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ben Browning
2026-06-15 11:59:05 +08:00
committed by GitHub
co-authored by Flora Feng
parent e3e3cd5458
commit c4a3f9d137
33 changed files with 8494 additions and 904 deletions
+115 -43
View File
@@ -23,8 +23,8 @@ from vllm.entrypoints.openai.engine.protocol import (
from vllm.parser.abstract_parser import DelegatingParser
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
from vllm.tool_parsers.qwen3coder_tool_parser import (
Qwen3CoderToolParser,
from vllm.tool_parsers.qwen3_engine_tool_parser import (
Qwen3EngineToolParser,
)
MODEL = "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
@@ -37,12 +37,7 @@ def qwen3_tokenizer():
@pytest.fixture
def qwen3_tool_parser(qwen3_tokenizer, sample_tools):
return Qwen3CoderToolParser(qwen3_tokenizer, tools=sample_tools)
@pytest.fixture
def qwen3_tool_parser_parametrized(qwen3_tool_parser):
return qwen3_tool_parser
return Qwen3EngineToolParser(qwen3_tokenizer, tools=sample_tools)
WEATHER_PARAMS = {
@@ -208,9 +203,9 @@ def stream_delta_message_generator(
read_offset = new_read_offset
def test_extract_tool_calls_no_tools(qwen3_tool_parser_parametrized):
def test_extract_tool_calls_no_tools(qwen3_tool_parser):
model_output = "This is a test response without any tool calls"
extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
model_output, request=None
) # type: ignore[arg-type]
assert not extracted_tool_calls.tools_called
@@ -391,13 +386,13 @@ circle
],
)
def test_extract_tool_calls(
qwen3_tool_parser_parametrized,
qwen3_tool_parser,
model_output,
expected_tool_calls,
expected_content,
):
request = ChatCompletionRequest(model=MODEL, messages=[])
extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
model_output, request=request
)
assert extracted_tool_calls.tools_called
@@ -408,7 +403,7 @@ def test_extract_tool_calls(
def test_extract_tool_calls_fallback_no_tags(
qwen3_tool_parser_parametrized,
qwen3_tool_parser,
):
"""Test fallback parsing when XML tags are missing"""
model_output = """<function=get_current_weather>
@@ -421,7 +416,7 @@ TX
</function>"""
request = ChatCompletionRequest(model=MODEL, messages=[])
extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
model_output, request=request
)
@@ -471,7 +466,7 @@ hello world
</function>
</tool_call>"""
parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)
@@ -563,7 +558,7 @@ some text
</function>
</tool_call>"""
parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
extracted = parser.extract_tool_calls(model_output, request=request)
@@ -637,7 +632,7 @@ true
</function>
</tool_call>"""
parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
tool_states = {}
@@ -843,7 +838,7 @@ circle
],
)
def test_extract_tool_calls_streaming(
qwen3_tool_parser_parametrized,
qwen3_tool_parser,
qwen3_tokenizer,
model_output,
expected_tool_calls,
@@ -856,7 +851,7 @@ def test_extract_tool_calls_streaming(
tool_states = {} # Track state per tool index
for delta_message in stream_delta_message_generator(
qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request
qwen3_tool_parser, qwen3_tokenizer, model_output, request
):
# role should never be streamed from tool parser
assert not delta_message.role
@@ -900,9 +895,6 @@ def test_extract_tool_calls_streaming(
# Verify we got all expected tool calls
assert len(tool_states) == len(expected_tool_calls)
assert len(qwen3_tool_parser_parametrized.prev_tool_call_arr) == len(
expected_tool_calls
)
# Verify each tool call
for idx, expected_tool in enumerate(expected_tool_calls):
@@ -920,7 +912,7 @@ def test_extract_tool_calls_streaming(
def test_extract_tool_calls_missing_closing_parameter_tag(
qwen3_tool_parser_parametrized,
qwen3_tool_parser,
):
"""Test handling of missing closing </parameter> tag"""
# Using get_current_weather from sample_tools but with malformed XML
@@ -939,7 +931,7 @@ fahrenheit
</tool_call>"""
request = ChatCompletionRequest(model=MODEL, messages=[])
extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
model_output, request=request
)
@@ -962,7 +954,7 @@ fahrenheit
def test_extract_tool_calls_streaming_missing_closing_tag(
qwen3_tool_parser_parametrized, qwen3_tokenizer
qwen3_tool_parser, qwen3_tokenizer
):
"""Test streaming with missing closing </parameter> tag"""
# Using get_current_weather from sample_tools but with malformed XML
@@ -986,7 +978,7 @@ fahrenheit
tool_states = {}
for delta_message in stream_delta_message_generator(
qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request
qwen3_tool_parser, qwen3_tokenizer, model_output, request
):
if delta_message.content:
other_content += delta_message.content
@@ -1021,7 +1013,6 @@ fahrenheit
assert "Let me check the weather for you:" in other_content
# Verify we got the tool call
assert len(tool_states) == 1
assert len(qwen3_tool_parser_parametrized.prev_tool_call_arr) == 1
state = tool_states[0]
assert state["id"] is not None
@@ -1036,9 +1027,7 @@ fahrenheit
assert args["unit"] == "fahrenheit"
def test_extract_tool_calls_streaming_incremental(
qwen3_tool_parser_parametrized, qwen3_tokenizer
):
def test_extract_tool_calls_streaming_incremental(qwen3_tool_parser, qwen3_tokenizer):
"""Test that streaming is truly incremental"""
model_output = """I'll check the weather.<tool_call>
<function=get_current_weather>
@@ -1055,7 +1044,7 @@ TX
chunks = []
for delta_message in stream_delta_message_generator(
qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request
qwen3_tool_parser, qwen3_tokenizer, model_output, request
):
chunks.append(delta_message)
@@ -1073,19 +1062,21 @@ TX
header_found = True
assert chunk.tool_calls[0].function.name == "get_current_weather"
assert chunk.tool_calls[0].type == "function"
# Empty initially
assert chunk.tool_calls[0].function.arguments == ""
break
assert header_found
# Should have chunks with incremental arguments
arg_chunks = []
for chunk in chunks:
if chunk.tool_calls and chunk.tool_calls[0].function.arguments:
if (
chunk.tool_calls
and chunk.tool_calls[0].function
and chunk.tool_calls[0].function.arguments
):
arg_chunks.append(chunk.tool_calls[0].function.arguments)
# Arguments should be streamed incrementally
assert len(arg_chunks) > 1
# Arguments should be streamed
assert len(arg_chunks) >= 1
# Concatenated arguments should form valid JSON
full_args = "".join(arg_chunks)
@@ -1094,6 +1085,85 @@ TX
assert parsed_args["state"] == "TX"
def test_extract_tool_calls_streaming_missing_opening_tag(
qwen3_tool_parser, qwen3_tokenizer
):
"""Test streaming with missing opening <tool_call> tag
This tests that the streaming parser correctly handles
tool calls that start directly with <function=...>
"""
model_output = """I'll check the weather for you.
<function=get_current_weather>
<parameter=city>
Dallas
</parameter>
<parameter=state>
TX
</parameter>
<parameter=unit>
fahrenheit
</parameter>
</function>
</tool_call>"""
request = ChatCompletionRequest(model=MODEL, messages=[])
other_content = ""
tool_states = {}
for delta_message in stream_delta_message_generator(
qwen3_tool_parser, qwen3_tokenizer, model_output, request
):
if delta_message.content:
other_content += delta_message.content
if delta_message.tool_calls:
for tool_call in delta_message.tool_calls:
idx = tool_call.index
if idx not in tool_states:
tool_states[idx] = {
"id": None,
"name": None,
"arguments": "",
"type": None,
}
if tool_call.id:
tool_states[idx]["id"] = tool_call.id
if tool_call.type:
assert tool_call.type == "function"
tool_states[idx]["type"] = tool_call.type
if tool_call.function:
if tool_call.function.name:
tool_states[idx]["name"] = tool_call.function.name
if tool_call.function.arguments is not None:
tool_states[idx]["arguments"] += tool_call.function.arguments
# Verify content was streamed
assert "I'll check the weather for you." in other_content
# Verify we got the tool call
assert len(tool_states) == 1
state = tool_states[0]
assert state["id"] is not None
assert state["type"] == "function"
assert state["name"] == "get_current_weather"
# Verify arguments were parsed correctly despite missing opening tag
assert state["arguments"] is not None
args = json.loads(state["arguments"])
assert args["city"] == "Dallas"
assert args["state"] == "TX"
assert args["unit"] == "fahrenheit"
def test_malformed_xml_no_gt_delimiter(qwen3_tool_parser):
"""Regression: malformed XML without '>' must not crash (PR #36774)."""
model_output = (
@@ -1130,9 +1200,11 @@ def test_none_tool_calls_filtered(qwen3_tool_parser):
result = qwen3_tool_parser.extract_tool_calls(model_output, request=request)
assert all(tc is not None for tc in result.tool_calls)
assert result.tools_called
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_current_weather"
args = json.loads(result.tool_calls[0].function.arguments)
valid = [
tc for tc in result.tool_calls if tc.function.name == "get_current_weather"
]
assert len(valid) == 1
args = json.loads(valid[0].function.arguments)
assert args["city"] == "Dallas"
assert args["state"] == "TX"
@@ -1156,7 +1228,7 @@ def test_anyof_parameter_not_double_encoded(qwen3_tokenizer):
)
]
parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
model_output = (
"<tool_call>\n"
@@ -1247,7 +1319,7 @@ def test_no_double_serialization_string_args(qwen3_tool_parser):
def test_get_vllm_registry_structural_tag_returns_structural_tag(
qwen3_tool_parser: Qwen3CoderToolParser,
qwen3_tool_parser: Qwen3EngineToolParser,
sample_tools: list[ChatCompletionToolsParam],
) -> None:
request_tools = _as_chat_completion_tools(sample_tools)
@@ -1289,7 +1361,7 @@ def test_adjust_request_auto_uses_vllm_registry_structural_tag(
include_reasoning: bool,
) -> None:
class TestParser(DelegatingParser):
tool_parser_cls = Qwen3CoderToolParser
tool_parser_cls = Qwen3EngineToolParser
request_tools = _as_chat_completion_tools(sample_tools)
req = ChatCompletionRequest(
@@ -1311,7 +1383,7 @@ def test_adjust_request_required_prefers_structural_tag(
sample_tools: list[ChatCompletionToolsParam],
) -> None:
class TestParser(DelegatingParser):
tool_parser_cls = Qwen3CoderToolParser
tool_parser_cls = Qwen3EngineToolParser
request_tools = _as_chat_completion_tools(sample_tools)
req = ChatCompletionRequest(