[Frontend] Add Streaming Parser Engine and new Kimi k2.5/k2.6/k2.7 Parser (#46610)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
8cc242335d
commit
2bc20e8aba
@@ -31,6 +31,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
Gemma4Parser,
|
||||
Glm47MoeParser,
|
||||
KimiK2Parser,
|
||||
MinimaxM2Parser,
|
||||
NemotronV3Parser,
|
||||
Qwen3Parser,
|
||||
@@ -717,6 +718,96 @@ def _build_glm47_moe(scenario: Scenario, validate: bool = True) -> Sample:
|
||||
return sample
|
||||
|
||||
|
||||
# ── Kimi K2 (native tool-call section, starts in REASONING) ──────────
|
||||
|
||||
_KIMI_K2_VOCAB: dict[str, int] = {
|
||||
"<think>": 50,
|
||||
"</think>": 51,
|
||||
"<|tool_calls_section_begin|>": 60,
|
||||
"<|tool_calls_section_end|>": 61,
|
||||
"<|tool_call_begin|>": 62,
|
||||
"<|tool_call_end|>": 63,
|
||||
"<|tool_call_argument_begin|>": 64,
|
||||
}
|
||||
|
||||
|
||||
def _kimi_k2_tool_segments(
|
||||
tool_calls: list[ToolCallSpec],
|
||||
) -> list[tuple[str, bool]]:
|
||||
segs: list[tuple[str, bool]] = [("<|tool_calls_section_begin|>", True)]
|
||||
for index, tc in enumerate(tool_calls):
|
||||
args = json.dumps(tc.arguments, ensure_ascii=False, separators=(",", ":"))
|
||||
segs.extend(
|
||||
[
|
||||
("<|tool_call_begin|>", True),
|
||||
(f"functions.{tc.name}:{index}\n", False),
|
||||
("<|tool_call_argument_begin|>", True),
|
||||
(args, False),
|
||||
("<|tool_call_end|>", True),
|
||||
]
|
||||
)
|
||||
segs.append(("<|tool_calls_section_end|>", True))
|
||||
return segs
|
||||
|
||||
|
||||
def _kimi_k2_segments(scenario: Scenario) -> list[tuple[str, bool]]:
|
||||
segs: list[tuple[str, bool]] = []
|
||||
if scenario.reasoning is not None:
|
||||
segs.append(("<think>", True))
|
||||
segs.append((scenario.reasoning, False))
|
||||
if scenario.content is not None or scenario.tool_calls is not None:
|
||||
segs.append(("</think>", True))
|
||||
if scenario.content is not None:
|
||||
segs.append((scenario.content, False))
|
||||
if scenario.tool_calls is not None:
|
||||
segs.extend(_kimi_k2_tool_segments(scenario.tool_calls))
|
||||
return segs
|
||||
|
||||
|
||||
def _build_kimi_k2(
|
||||
scenario: Scenario,
|
||||
validate: bool = True,
|
||||
thinking: bool = True,
|
||||
) -> Sample:
|
||||
expected_reasoning = (
|
||||
scenario.reasoning.rstrip()
|
||||
if (thinking and scenario.reasoning is not None)
|
||||
else None
|
||||
)
|
||||
if thinking and scenario.reasoning is None:
|
||||
expected_reasoning = ""
|
||||
|
||||
sample = _make_sample(
|
||||
sample_id=f"kimi_k2-{scenario.id}",
|
||||
description=scenario.description,
|
||||
vocab=_KIMI_K2_VOCAB,
|
||||
segments=_kimi_k2_segments(scenario),
|
||||
expected_reasoning=expected_reasoning,
|
||||
expected_content=_qwen3_expected_content(scenario),
|
||||
expected_tool_calls=_expected_tc(scenario),
|
||||
tools=_expected_tools(scenario),
|
||||
chat_template_kwargs=None if thinking else {"thinking": False},
|
||||
)
|
||||
if validate:
|
||||
_validate_sample(
|
||||
sample,
|
||||
KimiK2Parser,
|
||||
chat_template_kwargs=sample.chat_template_kwargs,
|
||||
)
|
||||
return sample
|
||||
|
||||
|
||||
_KIMI_K2_SCENARIOS = [
|
||||
*SCENARIOS,
|
||||
Scenario(
|
||||
id="trailing-reasoning-whitespace",
|
||||
description="Reasoning trailing whitespace is stripped",
|
||||
reasoning="Reasoning with trailing whitespace. \n\t",
|
||||
content="Done.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ── Registry and public API ──────────────────────────────────────────
|
||||
|
||||
_BUILDERS: dict[str, Any] = {
|
||||
@@ -726,6 +817,7 @@ _BUILDERS: dict[str, Any] = {
|
||||
"nemotron_v3": _build_nemotron_v3,
|
||||
"seed_oss": _build_seed_oss,
|
||||
"glm47_moe": _build_glm47_moe,
|
||||
"kimi_k2": _build_kimi_k2,
|
||||
}
|
||||
|
||||
|
||||
@@ -733,7 +825,8 @@ _BUILDERS: dict[str, Any] = {
|
||||
def build_samples(model: str) -> tuple[Sample, ...]:
|
||||
"""Build all scenario samples for a model, self-validated."""
|
||||
builder = _BUILDERS[model]
|
||||
return tuple(builder(s) for s in SCENARIOS)
|
||||
scenarios = _KIMI_K2_SCENARIOS if model == "kimi_k2" else SCENARIOS
|
||||
return tuple(builder(s) for s in scenarios)
|
||||
|
||||
|
||||
def build_sample(model: str, scenario: Scenario) -> Sample:
|
||||
|
||||
@@ -7,7 +7,6 @@ import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser
|
||||
from vllm.reasoning.kimi_k2_reasoning_parser import KimiK2ReasoningParser
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
@@ -33,20 +32,6 @@ def kimi_k2_tokenizer():
|
||||
return get_tokenizer(tokenizer_name=REASONING_MODEL_NAME, trust_remote_code=True)
|
||||
|
||||
|
||||
def test_parser_selection_thinking_enabled(kimi_k2_tokenizer):
|
||||
parser = KimiK2ReasoningParser(
|
||||
kimi_k2_tokenizer, chat_template_kwargs={"thinking": True}
|
||||
)
|
||||
assert parser._identity_parser is None
|
||||
|
||||
|
||||
def test_parser_selection_thinking_disabled(kimi_k2_tokenizer):
|
||||
parser = KimiK2ReasoningParser(
|
||||
kimi_k2_tokenizer, chat_template_kwargs={"thinking": False}
|
||||
)
|
||||
assert isinstance(parser._identity_parser, IdentityReasoningParser)
|
||||
|
||||
|
||||
def test_extract_reasoning_with_think_tags(kimi_k2_tokenizer):
|
||||
parser = KimiK2ReasoningParser(kimi_k2_tokenizer)
|
||||
request = ChatCompletionRequest(model="test-model", messages=[], temperature=1.0)
|
||||
@@ -65,7 +50,7 @@ def test_extract_reasoning_empty_thinking(kimi_k2_tokenizer):
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<think></think>final answer", request
|
||||
)
|
||||
assert reasoning == ""
|
||||
assert reasoning is None
|
||||
assert content == "final answer"
|
||||
|
||||
|
||||
@@ -96,8 +81,8 @@ def test_streaming_reasoning_then_content(kimi_k2_tokenizer):
|
||||
"""Token-by-token streaming: reasoning tokens then content after </think>."""
|
||||
parser = KimiK2ReasoningParser(kimi_k2_tokenizer)
|
||||
|
||||
think_id = parser._start_token_id
|
||||
end_think_id = parser._end_token_id
|
||||
think_id = parser._parser_engine._start_token_id
|
||||
end_think_id = parser._parser_engine._end_token_id
|
||||
# Use a real token ID from the tokenizer for regular content
|
||||
regular_id = kimi_k2_tokenizer.encode("hello", add_special_tokens=False)[0]
|
||||
|
||||
@@ -154,8 +139,8 @@ def test_streaming_tool_section_ends_reasoning(kimi_k2_tokenizer):
|
||||
"""<|tool_calls_section_begin|> in delta ends reasoning during streaming."""
|
||||
parser = KimiK2ReasoningParser(kimi_k2_tokenizer)
|
||||
|
||||
think_id = parser._start_token_id
|
||||
tool_begin_id = parser._tool_section_start_token_id
|
||||
think_id = parser._parser_engine._start_token_id
|
||||
tool_begin_id = parser._parser_engine._tool_section_start_token_id
|
||||
regular_id = kimi_k2_tokenizer.encode("hello", add_special_tokens=False)[0]
|
||||
|
||||
# Tool section token arrives — should transition from reasoning to content
|
||||
@@ -169,50 +154,3 @@ def test_streaming_tool_section_ends_reasoning(kimi_k2_tokenizer):
|
||||
)
|
||||
assert isinstance(result, DeltaMessage)
|
||||
assert result.content == "<|tool_calls_section_begin|>"
|
||||
|
||||
|
||||
def test_streaming_end_token_id_buffered(mock_kimi_k2_tokenizer):
|
||||
"""When stop sequences buffer text, </think> ID arrives before its text.
|
||||
|
||||
The token ID is present in delta_token_ids but the actual string is not
|
||||
yet in delta_text (still buffered). The parser must return None to wait
|
||||
for the next delta, instead of calling find() which returns -1 and
|
||||
silently corrupting the text split.
|
||||
"""
|
||||
parser = KimiK2ReasoningParser(mock_kimi_k2_tokenizer)
|
||||
think_id = parser._start_token_id
|
||||
end_think_id = parser._end_token_id
|
||||
|
||||
# Simulate: </think> ID arrived but text not yet flushed.
|
||||
# Two token IDs in delta to bypass the single-special-token guard.
|
||||
result = parser.extract_reasoning_streaming(
|
||||
previous_text="some reasoning",
|
||||
current_text="some reasoning extra",
|
||||
delta_text="extra", # </think> text not yet flushed
|
||||
previous_token_ids=[think_id],
|
||||
current_token_ids=[think_id, end_think_id, 999],
|
||||
delta_token_ids=[end_think_id, 999],
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_streaming_tool_section_id_buffered(mock_kimi_k2_tokenizer):
|
||||
"""When stop sequences buffer text, tool section start ID arrives before its text.
|
||||
|
||||
Same buffering scenario as above but for <|tool_calls_section_begin|>.
|
||||
Without the guard, find() returns -1 and delta_text[:tool_index] silently
|
||||
drops the last character of reasoning.
|
||||
"""
|
||||
parser = KimiK2ReasoningParser(mock_kimi_k2_tokenizer)
|
||||
think_id = parser._start_token_id
|
||||
tool_begin_id = parser._tool_section_start_token_id
|
||||
|
||||
result = parser.extract_reasoning_streaming(
|
||||
previous_text="some reasoning",
|
||||
current_text="some reasoning extra",
|
||||
delta_text="extra", # tool section text not yet flushed
|
||||
previous_token_ids=[think_id],
|
||||
current_token_ids=[think_id, tool_begin_id, 999],
|
||||
delta_token_ids=[tool_begin_id, 999],
|
||||
)
|
||||
assert result is None
|
||||
|
||||
Reference in New Issue
Block a user