chore: Upstream Cohere parser fixes + tests (#51998)
Signed-off-by: Jason Ozuzu <jasonozuzu@cohere.com>
This commit is contained in:
@@ -126,7 +126,7 @@ cloudpickle==3.1.2
|
||||
# via -r requirements/test/../common.txt
|
||||
cohere==7.0.8
|
||||
# via -r requirements/test/cuda.in
|
||||
cohere-melody==0.9.0
|
||||
cohere-melody==0.11.1
|
||||
# via -r requirements/test/cuda.in
|
||||
colorama==0.4.6
|
||||
# via
|
||||
|
||||
@@ -71,7 +71,7 @@ gpt-oss >= 0.0.7; python_version > '3.11'
|
||||
|
||||
perceptron # required for isaac test
|
||||
kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test
|
||||
cohere_melody>=0.9.0 # required for cohere command reasoning parser test
|
||||
cohere_melody>=0.11.1 # required for cohere command reasoning/tool parser tests
|
||||
cohere>=7.0.0 # required for cohere chat v2 api tests (protocol/serving import cohere.types)
|
||||
|
||||
# Newer versions of datasets require torchcoded, that makes the tests fail in CI because of a missing library.
|
||||
|
||||
@@ -131,7 +131,7 @@ cloudpickle==3.1.2
|
||||
# via -r requirements/test/../common.txt
|
||||
cohere==7.0.8
|
||||
# via -r requirements/test/cuda.in
|
||||
cohere-melody==0.9.0
|
||||
cohere-melody==0.11.1
|
||||
# via -r requirements/test/cuda.in
|
||||
colorama==0.4.6
|
||||
# via
|
||||
|
||||
@@ -69,7 +69,7 @@ gpt-oss>=0.0.7; python_version > '3.11'
|
||||
|
||||
perceptron # required for isaac test
|
||||
kaldi-native-fbank>=1.18.7 # required for fireredasr2 test
|
||||
cohere_melody>=0.9.0 # required for cohere command reasoning parser test
|
||||
cohere_melody>=0.11.1 # required for cohere command reasoning/tool parser tests
|
||||
cohere>=7.0.0 # required for cohere chat v2 api tests (protocol/serving import cohere.types)
|
||||
|
||||
# Newer versions of datasets require torchcoded, that makes the tests fail in CI because of a missing library.
|
||||
|
||||
@@ -131,7 +131,7 @@ cloudpickle==3.1.2
|
||||
# tilelang
|
||||
cohere==7.0.8
|
||||
# via -r requirements/test/rocm.in
|
||||
cohere-melody==0.9.0
|
||||
cohere-melody==0.11.1
|
||||
# via -r requirements/test/rocm.in
|
||||
colorama==0.4.6
|
||||
# via
|
||||
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import UserDict
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
@@ -34,20 +34,12 @@ from vllm.reasoning.cohere_command_reasoning_parser import (
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExpectedToolCall:
|
||||
id: str
|
||||
name: str
|
||||
arguments: dict
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReasoningCase:
|
||||
parser_cls: Any
|
||||
model_output: str
|
||||
expected_reasoning: str | None
|
||||
expected_content: str | None
|
||||
expected_tool_calls: list[ExpectedToolCall] = field(default_factory=list)
|
||||
|
||||
|
||||
REASONING_CASES = [
|
||||
@@ -67,9 +59,6 @@ REASONING_CASES = [
|
||||
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
|
||||
]
|
||||
<|END_ACTION|>""",
|
||||
expected_tool_calls=[
|
||||
ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
|
||||
],
|
||||
),
|
||||
id="cmd3-single_tool_call",
|
||||
),
|
||||
@@ -89,9 +78,6 @@ REASONING_CASES = [
|
||||
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
|
||||
]
|
||||
<|END_ACTION|>""",
|
||||
expected_tool_calls=[
|
||||
ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
|
||||
],
|
||||
),
|
||||
id="cmd4-single_tool_call",
|
||||
),
|
||||
@@ -236,29 +222,8 @@ class TestExtractReasoning:
|
||||
assert reasoning == case.expected_reasoning
|
||||
|
||||
content = "".join(content_parts) if content_parts else None
|
||||
if case.expected_tool_calls:
|
||||
assert content is None or content == ""
|
||||
else:
|
||||
assert content == case.expected_content
|
||||
|
||||
accumulated: dict[int, dict] = {}
|
||||
for d in tool_call_deltas:
|
||||
idx = d["index"]
|
||||
if idx not in accumulated:
|
||||
accumulated[idx] = {"id": "", "name": "", "arguments": ""}
|
||||
if d["id"]:
|
||||
accumulated[idx]["id"] = d["id"]
|
||||
if d["name"]:
|
||||
accumulated[idx]["name"] = d["name"]
|
||||
if d["arguments"]:
|
||||
accumulated[idx]["arguments"] += d["arguments"]
|
||||
|
||||
assert len(accumulated) == len(case.expected_tool_calls)
|
||||
for i, expected_tc in enumerate(case.expected_tool_calls):
|
||||
tc = accumulated[i]
|
||||
assert tc["id"] == expected_tc.id
|
||||
assert tc["name"] == expected_tc.name
|
||||
assert json.loads(tc["arguments"]) == expected_tc.arguments
|
||||
assert content == case.expected_content
|
||||
assert tool_call_deltas == []
|
||||
|
||||
|
||||
class TestIsReasoningEnd:
|
||||
|
||||
@@ -0,0 +1,510 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tool_parsers.cohere_command_tool_parser import (
|
||||
CohereCommand3ToolParser,
|
||||
CohereCommand4ToolParser,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExpectedToolCall:
|
||||
id: str
|
||||
name: str
|
||||
arguments: dict
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallCase:
|
||||
parser_cls: Any
|
||||
model_output: str
|
||||
expected_tool_calls: list[ExpectedToolCall] = field(default_factory=list)
|
||||
expected_reasoning: str | None = None
|
||||
expected_content: str | None = None
|
||||
|
||||
|
||||
TOOL_CALL_CASES = [
|
||||
pytest.param(
|
||||
ToolCallCase(
|
||||
parser_cls=CohereCommand3ToolParser,
|
||||
model_output="""\
|
||||
<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
|
||||
[
|
||||
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
|
||||
]
|
||||
<|END_ACTION|>""",
|
||||
expected_tool_calls=[
|
||||
ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
|
||||
],
|
||||
expected_reasoning="i will call foo with query1",
|
||||
),
|
||||
id="cmd3-single_tool_call",
|
||||
),
|
||||
pytest.param(
|
||||
ToolCallCase(
|
||||
parser_cls=CohereCommand4ToolParser,
|
||||
model_output="""\
|
||||
<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
|
||||
[
|
||||
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
|
||||
]
|
||||
<|END_ACTION|>""",
|
||||
expected_tool_calls=[
|
||||
ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
|
||||
],
|
||||
expected_reasoning="i will call foo with query1",
|
||||
),
|
||||
id="cmd4-single_tool_call",
|
||||
),
|
||||
pytest.param(
|
||||
ToolCallCase(
|
||||
parser_cls=CohereCommand3ToolParser,
|
||||
model_output="""\
|
||||
<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>
|
||||
<|START_RESPONSE|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_RESPONSE|>""",
|
||||
expected_reasoning="This is a rainbow emoji: 🌈",
|
||||
expected_content="foo bar",
|
||||
),
|
||||
id="cmd3-citations_no_tool_calls",
|
||||
),
|
||||
pytest.param(
|
||||
ToolCallCase(
|
||||
parser_cls=CohereCommand4ToolParser,
|
||||
model_output="""\
|
||||
<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>
|
||||
<|START_RESPONSE|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_RESPONSE|>""",
|
||||
expected_reasoning="This is a rainbow emoji: 🌈",
|
||||
expected_content="foo bar",
|
||||
),
|
||||
id="cmd4-citations_no_tool_calls",
|
||||
),
|
||||
pytest.param(
|
||||
ToolCallCase(
|
||||
parser_cls=CohereCommand3ToolParser,
|
||||
model_output="""\
|
||||
<|START_THINKING|>first I think about foo<|END_THINKING|><|START_ACTION|>
|
||||
[
|
||||
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}},
|
||||
{"tool_call_id": "1", "tool_name": "bar", "parameters": {"x": 42}}
|
||||
]
|
||||
<|END_ACTION|>""",
|
||||
expected_tool_calls=[
|
||||
ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
|
||||
ExpectedToolCall(id="1", name="bar", arguments={"x": 42}),
|
||||
],
|
||||
expected_reasoning="first I think about foo",
|
||||
),
|
||||
id="cmd3-multiple_tool_calls",
|
||||
),
|
||||
pytest.param(
|
||||
ToolCallCase(
|
||||
parser_cls=CohereCommand4ToolParser,
|
||||
model_output="""\
|
||||
<|START_THINKING|>first I think about foo<|END_THINKING|><|START_ACTION|>
|
||||
[
|
||||
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}},
|
||||
{"tool_call_id": "1", "tool_name": "bar", "parameters": {"x": 42}}
|
||||
]
|
||||
<|END_ACTION|>""",
|
||||
expected_tool_calls=[
|
||||
ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}),
|
||||
ExpectedToolCall(id="1", name="bar", arguments={"x": 42}),
|
||||
],
|
||||
expected_reasoning="first I think about foo",
|
||||
),
|
||||
id="cmd4-multiple_tool_calls",
|
||||
),
|
||||
pytest.param(
|
||||
ToolCallCase(
|
||||
parser_cls=CohereCommand3ToolParser,
|
||||
model_output="""\
|
||||
<|START_THINKING|>just think, no response<|END_THINKING|>""",
|
||||
expected_reasoning="just think, no response",
|
||||
),
|
||||
id="cmd3-reasoning_only",
|
||||
),
|
||||
pytest.param(
|
||||
ToolCallCase(
|
||||
parser_cls=CohereCommand4ToolParser,
|
||||
model_output="""\
|
||||
<|START_THINKING|>just think, no response<|END_THINKING|>""",
|
||||
expected_reasoning="just think, no response",
|
||||
),
|
||||
id="cmd4-reasoning_only",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class MockCohereTokenizer:
|
||||
"""Minimal byte-level stand-in for the Cohere tokenizer."""
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
|
||||
return list(text.encode("utf-8"))
|
||||
|
||||
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
|
||||
return bytes(ids).decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer() -> MockCohereTokenizer:
|
||||
return MockCohereTokenizer()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def request_obj() -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
|
||||
REPLACEMENT_CHAR = "\ufffd"
|
||||
|
||||
|
||||
def _token_deltas(tokenizer: MockCohereTokenizer, text: str) -> list[str]:
|
||||
"""Decode per-token string deltas, buffering incomplete multi-byte chars."""
|
||||
ids = tokenizer.encode(text, add_special_tokens=False)
|
||||
deltas: list[str] = []
|
||||
prev = ""
|
||||
for i in range(1, len(ids) + 1):
|
||||
current = tokenizer.decode(ids[:i], skip_special_tokens=False)
|
||||
if current.endswith(REPLACEMENT_CHAR):
|
||||
continue
|
||||
delta = current[len(prev) :]
|
||||
if delta:
|
||||
deltas.append(delta)
|
||||
prev = current
|
||||
return deltas
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamingResult:
|
||||
tool_calls: dict[int, dict]
|
||||
reasoning: str | None
|
||||
content: str | None
|
||||
|
||||
|
||||
def _run_streaming_over_deltas(
|
||||
parser,
|
||||
deltas: list[str],
|
||||
request_obj: ChatCompletionRequest,
|
||||
) -> StreamingResult:
|
||||
accumulated: dict[int, dict] = {}
|
||||
reasoning_parts: list[str] = []
|
||||
content_parts: list[str] = []
|
||||
previous_text = ""
|
||||
|
||||
for token_str in deltas:
|
||||
current_text = previous_text + token_str
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=token_str,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=request_obj,
|
||||
)
|
||||
if delta is not None:
|
||||
if delta.reasoning is not None:
|
||||
reasoning_parts.append(delta.reasoning)
|
||||
if delta.content is not None:
|
||||
content_parts.append(delta.content)
|
||||
for tc in delta.tool_calls:
|
||||
idx = tc.index
|
||||
if idx not in accumulated:
|
||||
accumulated[idx] = {"id": "", "name": "", "arguments": ""}
|
||||
if tc.id:
|
||||
accumulated[idx]["id"] = tc.id
|
||||
if tc.function and tc.function.name:
|
||||
accumulated[idx]["name"] = tc.function.name
|
||||
if tc.function and tc.function.arguments:
|
||||
accumulated[idx]["arguments"] += tc.function.arguments
|
||||
previous_text = current_text
|
||||
|
||||
return StreamingResult(
|
||||
tool_calls=accumulated,
|
||||
reasoning="".join(reasoning_parts) if reasoning_parts else None,
|
||||
content="".join(content_parts) if content_parts else None,
|
||||
)
|
||||
|
||||
|
||||
def _run_streaming(
|
||||
parser,
|
||||
tokenizer: MockCohereTokenizer,
|
||||
model_output: str,
|
||||
request_obj: ChatCompletionRequest,
|
||||
) -> StreamingResult:
|
||||
return _run_streaming_over_deltas(
|
||||
parser,
|
||||
_token_deltas(tokenizer, model_output),
|
||||
request_obj,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", TOOL_CALL_CASES)
|
||||
class TestExtractToolCalls:
|
||||
def test_streaming(
|
||||
self,
|
||||
tokenizer: MockCohereTokenizer,
|
||||
request_obj: ChatCompletionRequest,
|
||||
case: ToolCallCase,
|
||||
):
|
||||
parser = case.parser_cls(tokenizer)
|
||||
streamed = _run_streaming(parser, tokenizer, case.model_output, request_obj)
|
||||
|
||||
assert len(streamed.tool_calls) == len(case.expected_tool_calls)
|
||||
|
||||
for i, expected_tc in enumerate(case.expected_tool_calls):
|
||||
tc = streamed.tool_calls[i]
|
||||
assert tc["id"] == expected_tc.id
|
||||
assert tc["name"] == expected_tc.name
|
||||
assert json.loads(tc["arguments"]) == expected_tc.arguments
|
||||
|
||||
def test_streaming_reasoning(
|
||||
self,
|
||||
tokenizer: MockCohereTokenizer,
|
||||
request_obj: ChatCompletionRequest,
|
||||
case: ToolCallCase,
|
||||
):
|
||||
parser = case.parser_cls(tokenizer)
|
||||
streamed = _run_streaming(parser, tokenizer, case.model_output, request_obj)
|
||||
|
||||
assert streamed.reasoning == case.expected_reasoning
|
||||
|
||||
def test_streaming_content(
|
||||
self,
|
||||
tokenizer: MockCohereTokenizer,
|
||||
request_obj: ChatCompletionRequest,
|
||||
case: ToolCallCase,
|
||||
):
|
||||
parser = case.parser_cls(tokenizer)
|
||||
streamed = _run_streaming(parser, tokenizer, case.model_output, request_obj)
|
||||
|
||||
assert streamed.content == case.expected_content
|
||||
|
||||
def test_nonstreaming(
|
||||
self,
|
||||
request_obj: ChatCompletionRequest,
|
||||
tokenizer: MockCohereTokenizer,
|
||||
case: ToolCallCase,
|
||||
):
|
||||
parser = case.parser_cls(tokenizer)
|
||||
result = parser.extract_tool_calls(case.model_output, request_obj)
|
||||
|
||||
assert result.tools_called == (len(case.expected_tool_calls) > 0)
|
||||
assert len(result.tool_calls) == len(case.expected_tool_calls)
|
||||
|
||||
for actual_tc, expected_tc in zip(result.tool_calls, case.expected_tool_calls):
|
||||
assert actual_tc.type == "function"
|
||||
assert actual_tc.function.name == expected_tc.name
|
||||
assert json.loads(actual_tc.function.arguments) == expected_tc.arguments
|
||||
|
||||
def test_streaming_nonstreaming_agree(
|
||||
self,
|
||||
tokenizer: MockCohereTokenizer,
|
||||
request_obj: ChatCompletionRequest,
|
||||
case: ToolCallCase,
|
||||
):
|
||||
parser_streaming = case.parser_cls(tokenizer)
|
||||
parser_nonstreaming = case.parser_cls(tokenizer)
|
||||
|
||||
streamed = _run_streaming(
|
||||
parser_streaming,
|
||||
tokenizer,
|
||||
case.model_output,
|
||||
request_obj,
|
||||
)
|
||||
result = parser_nonstreaming.extract_tool_calls(
|
||||
case.model_output,
|
||||
request_obj,
|
||||
)
|
||||
|
||||
assert len(streamed.tool_calls) == len(result.tool_calls)
|
||||
|
||||
for i, actual_tc in enumerate(result.tool_calls):
|
||||
assert streamed.tool_calls[i]["name"] == actual_tc.function.name
|
||||
assert json.loads(streamed.tool_calls[i]["arguments"]) == json.loads(
|
||||
actual_tc.function.arguments
|
||||
)
|
||||
|
||||
|
||||
SPECIAL_TOKEN_MARKERS = (
|
||||
"<|START_THINKING|>",
|
||||
"<|END_THINKING|>",
|
||||
"<|START_RESPONSE|>",
|
||||
"<|END_RESPONSE|>",
|
||||
"<|START_ACTION|>",
|
||||
"<|END_ACTION|>",
|
||||
"<|START_TEXT|>",
|
||||
"<|END_TEXT|>",
|
||||
)
|
||||
|
||||
|
||||
def _multi_token_deltas(
|
||||
tokenizer: MockCohereTokenizer,
|
||||
text: str,
|
||||
chunk_size: int,
|
||||
) -> list[str]:
|
||||
ids = tokenizer.encode(text, add_special_tokens=False)
|
||||
deltas: list[str] = []
|
||||
prev = ""
|
||||
i = 0
|
||||
while i < len(ids):
|
||||
end = min(len(ids), i + chunk_size)
|
||||
current = tokenizer.decode(ids[:end], skip_special_tokens=False)
|
||||
i = end
|
||||
if current.endswith(REPLACEMENT_CHAR):
|
||||
continue
|
||||
delta = current[len(prev) :]
|
||||
if delta:
|
||||
deltas.append(delta)
|
||||
prev = current
|
||||
return deltas
|
||||
|
||||
|
||||
class TestSpeculativeDecodingMultiTokenDelta:
|
||||
MODEL_OUTPUT = (
|
||||
"<|START_THINKING|> i will call foo with query1<|END_THINKING|>"
|
||||
"<|START_ACTION|>\n"
|
||||
'[\n {"tool_call_id": "0", "tool_name": "foo", '
|
||||
'"parameters": {"query": "query1"}}\n]\n'
|
||||
"<|END_ACTION|>"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls",
|
||||
[CohereCommand3ToolParser, CohereCommand4ToolParser],
|
||||
ids=["cmd3", "cmd4"],
|
||||
)
|
||||
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 6])
|
||||
def test_no_special_token_leak_in_streaming_deltas(
|
||||
self,
|
||||
tokenizer: MockCohereTokenizer,
|
||||
request_obj: ChatCompletionRequest,
|
||||
parser_cls,
|
||||
chunk_size: int,
|
||||
):
|
||||
parser = parser_cls(tokenizer)
|
||||
chunked = _multi_token_deltas(tokenizer, self.MODEL_OUTPUT, chunk_size)
|
||||
|
||||
previous_text = ""
|
||||
for token_str in chunked:
|
||||
current_text = previous_text + token_str
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=token_str,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=request_obj,
|
||||
)
|
||||
previous_text = current_text
|
||||
if delta is None:
|
||||
continue
|
||||
|
||||
fields: list[tuple[str, str | None]] = [
|
||||
("reasoning", delta.reasoning),
|
||||
("content", delta.content),
|
||||
]
|
||||
for tc in delta.tool_calls or []:
|
||||
if tc.function:
|
||||
fields.append(("tool_call.name", tc.function.name))
|
||||
fields.append(("tool_call.arguments", tc.function.arguments))
|
||||
|
||||
for marker in SPECIAL_TOKEN_MARKERS:
|
||||
for field_name, value in fields:
|
||||
assert value is None or marker not in value, (
|
||||
f"special token {marker!r} leaked into {field_name} "
|
||||
f"with chunk_size={chunk_size} delta={delta!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls",
|
||||
[CohereCommand3ToolParser, CohereCommand4ToolParser],
|
||||
ids=["cmd3", "cmd4"],
|
||||
)
|
||||
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 6])
|
||||
def test_multi_token_chunks_still_produce_correct_tool_call(
|
||||
self,
|
||||
tokenizer: MockCohereTokenizer,
|
||||
request_obj: ChatCompletionRequest,
|
||||
parser_cls,
|
||||
chunk_size: int,
|
||||
):
|
||||
parser = parser_cls(tokenizer)
|
||||
chunked = _multi_token_deltas(tokenizer, self.MODEL_OUTPUT, chunk_size)
|
||||
streamed = _run_streaming_over_deltas(parser, chunked, request_obj)
|
||||
|
||||
assert len(streamed.tool_calls) == 1
|
||||
tc = streamed.tool_calls[0]
|
||||
assert tc["id"] == "0"
|
||||
assert tc["name"] == "foo"
|
||||
assert json.loads(tc["arguments"]) == {"query": "query1"}
|
||||
|
||||
|
||||
class TestStreamingDeltaShape:
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls",
|
||||
[CohereCommand3ToolParser, CohereCommand4ToolParser],
|
||||
ids=["cmd3", "cmd4"],
|
||||
)
|
||||
def test_reasoning_and_tool_calls_are_separate_deltas(
|
||||
self,
|
||||
tokenizer: MockCohereTokenizer,
|
||||
request_obj: ChatCompletionRequest,
|
||||
parser_cls,
|
||||
):
|
||||
parser = parser_cls(tokenizer)
|
||||
model_output = (
|
||||
"<|START_THINKING|> i will call foo with query1<|END_THINKING|>"
|
||||
"<|START_ACTION|>\n"
|
||||
'[\n {"tool_call_id": "0", "tool_name": "foo", '
|
||||
'"parameters": {"query": "query1"}}\n]\n'
|
||||
"<|END_ACTION|>"
|
||||
)
|
||||
|
||||
token_strings = _token_deltas(tokenizer, model_output)
|
||||
previous_text = ""
|
||||
saw_reasoning = False
|
||||
saw_tool_call = False
|
||||
|
||||
for token_str in token_strings:
|
||||
current_text = previous_text + token_str
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=token_str,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=request_obj,
|
||||
)
|
||||
if delta is not None:
|
||||
populated = [
|
||||
delta.content is not None,
|
||||
delta.reasoning is not None,
|
||||
bool(delta.tool_calls),
|
||||
]
|
||||
assert sum(populated) == 1, (
|
||||
"A single streaming delta must carry exactly one of "
|
||||
f"content/reasoning/tool_calls, got {delta!r}"
|
||||
)
|
||||
if delta.reasoning is not None:
|
||||
saw_reasoning = True
|
||||
if delta.tool_calls:
|
||||
saw_tool_call = True
|
||||
previous_text = current_text
|
||||
|
||||
assert saw_reasoning, "expected at least one reasoning delta"
|
||||
assert saw_tool_call, "expected at least one tool-call delta"
|
||||
@@ -16,7 +16,7 @@ except ImportError as e:
|
||||
raise ImportError(
|
||||
"The Cohere reasoning parser requires the `cohere_melody` "
|
||||
"package, which is not installed. Install it with:\n"
|
||||
" pip install cohere_melody"
|
||||
" pip install 'cohere-melody>=0.11.1'"
|
||||
) from e
|
||||
|
||||
|
||||
@@ -132,7 +132,7 @@ def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str:
|
||||
tool_dictionary[tool_name] = f"{tool_name} ::= {tool_name}root\n{tool_grammar}"
|
||||
# Emitted grammar shape:
|
||||
# root ::= tools
|
||||
# tools ::= ws "[" ws tool ws ("," ws tool)* ws "]" ws
|
||||
# tools ::= ws "[" ws tool (ws "," ws tool)* ws "]" ws
|
||||
# ws ::= (" " | "\t" | "\n")*
|
||||
# tool ::= <tool_a> | <tool_b> | ... (one alternative per input)
|
||||
# <tool_x> ::= <tool_x>root (per-tool xgrammar rules)
|
||||
@@ -140,7 +140,7 @@ def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str:
|
||||
tool_alternatives = "tool ::= " + " | ".join(tool_dictionary.keys())
|
||||
tool_rules = "\n ".join(tool_dictionary.values())
|
||||
grammar = f"""root ::= tools
|
||||
tools ::= ws "[" ws tool ws ("," ws tool)* ws "]" ws
|
||||
tools ::= ws "[" ws tool (ws "," ws tool)* ws "]" ws
|
||||
ws ::= (" " | "\\t" | "\\n")*
|
||||
{tool_alternatives}
|
||||
{tool_rules}
|
||||
@@ -699,7 +699,9 @@ class CohereCommand3ReasoningParser(BaseCohereCommandReasoningParser):
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
*args,
|
||||
streaming_opts=PyFilterOptions().cmd3().stream_non_grounded_answer(),
|
||||
streaming_opts=(
|
||||
PyFilterOptions().cmd3().no_tools().stream_non_grounded_answer()
|
||||
),
|
||||
unary_opts=PyFilterOptions().cmd3().no_tools(),
|
||||
**kwargs,
|
||||
)
|
||||
@@ -710,7 +712,9 @@ class CohereCommand4ReasoningParser(BaseCohereCommandReasoningParser):
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
*args,
|
||||
streaming_opts=PyFilterOptions().cmd4().stream_non_grounded_answer(),
|
||||
streaming_opts=(
|
||||
PyFilterOptions().cmd4().no_tools().stream_non_grounded_answer()
|
||||
),
|
||||
unary_opts=PyFilterOptions().cmd4().no_tools(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -209,7 +209,7 @@ def _try_import_melody():
|
||||
raise ImportError(
|
||||
"The `cohere` tokenizer/renderer mode requires the "
|
||||
"`cohere_melody` package. Install it via "
|
||||
"`pip install cohere-melody` or build from "
|
||||
"`pip install 'cohere-melody>=0.11.1'` or build from "
|
||||
"https://github.com/cohere-ai/melody."
|
||||
) from e
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ except ImportError as e:
|
||||
raise ImportError(
|
||||
"The Cohere tool parser requires the `cohere_melody` "
|
||||
"package, which is not installed. Install it with:\n"
|
||||
" pip install cohere_melody"
|
||||
" pip install 'cohere-melody>=0.11.1'"
|
||||
) from e
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
@@ -130,8 +130,8 @@ class CohereCommand3ToolParser(BaseCohereCommandToolParser):
|
||||
):
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
streaming_opts=PyFilterOptions().cmd3(),
|
||||
unary_opts=PyFilterOptions().cmd3(),
|
||||
streaming_opts=PyFilterOptions().cmd3().start_in_answer(),
|
||||
unary_opts=PyFilterOptions().cmd3().start_in_answer(),
|
||||
)
|
||||
|
||||
|
||||
@@ -143,6 +143,6 @@ class CohereCommand4ToolParser(BaseCohereCommandToolParser):
|
||||
):
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
streaming_opts=PyFilterOptions().cmd4(),
|
||||
unary_opts=PyFilterOptions().cmd4(),
|
||||
streaming_opts=PyFilterOptions().cmd4().start_in_answer(),
|
||||
unary_opts=PyFilterOptions().cmd4().start_in_answer(),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user