diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index cb45f29ed5..741e33dc02 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -126,7 +126,7 @@ cloudpickle==3.1.2 # via -r requirements/test/../common.txt cohere==7.0.8 # via -r requirements/test/cuda.in -cohere-melody==0.9.0 +cohere-melody==0.11.1 # via -r requirements/test/cuda.in colorama==0.4.6 # via diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index 8a56743b1c..d11da082ad 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -71,7 +71,7 @@ gpt-oss >= 0.0.7; python_version > '3.11' perceptron # required for isaac test kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test -cohere_melody>=0.9.0 # required for cohere command reasoning parser test +cohere_melody>=0.11.1 # required for cohere command reasoning/tool parser tests cohere>=7.0.0 # required for cohere chat v2 api tests (protocol/serving import cohere.types) # Newer versions of datasets require torchcoded, that makes the tests fail in CI because of a missing library. diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index f0bea6e5ba..fa3597b770 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -131,7 +131,7 @@ cloudpickle==3.1.2 # via -r requirements/test/../common.txt cohere==7.0.8 # via -r requirements/test/cuda.in -cohere-melody==0.9.0 +cohere-melody==0.11.1 # via -r requirements/test/cuda.in colorama==0.4.6 # via diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index 5858a028ca..a7986b6e04 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -69,7 +69,7 @@ gpt-oss>=0.0.7; python_version > '3.11' perceptron # required for isaac test kaldi-native-fbank>=1.18.7 # required for fireredasr2 test -cohere_melody>=0.9.0 # required for cohere command reasoning parser test +cohere_melody>=0.11.1 # required for cohere command reasoning/tool parser tests cohere>=7.0.0 # required for cohere chat v2 api tests (protocol/serving import cohere.types) # Newer versions of datasets require torchcoded, that makes the tests fail in CI because of a missing library. diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 83f491a257..a7b7172ac2 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -131,7 +131,7 @@ cloudpickle==3.1.2 # tilelang cohere==7.0.8 # via -r requirements/test/rocm.in -cohere-melody==0.9.0 +cohere-melody==0.11.1 # via -r requirements/test/rocm.in colorama==0.4.6 # via diff --git a/tests/reasoning/test_cohere_command_reasoning_parser.py b/tests/reasoning/test_cohere_command_reasoning_parser.py index 84bc40afe8..0c1fc89009 100644 --- a/tests/reasoning/test_cohere_command_reasoning_parser.py +++ b/tests/reasoning/test_cohere_command_reasoning_parser.py @@ -5,7 +5,7 @@ from __future__ import annotations import json from collections import UserDict -from dataclasses import dataclass, field +from dataclasses import dataclass from types import SimpleNamespace from typing import Any @@ -34,20 +34,12 @@ from vllm.reasoning.cohere_command_reasoning_parser import ( from vllm.sampling_params import StructuredOutputsParams -@dataclass -class ExpectedToolCall: - id: str - name: str - arguments: dict - - @dataclass class ReasoningCase: parser_cls: Any model_output: str expected_reasoning: str | None expected_content: str | None - expected_tool_calls: list[ExpectedToolCall] = field(default_factory=list) REASONING_CASES = [ @@ -67,9 +59,6 @@ REASONING_CASES = [ {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}} ] <|END_ACTION|>""", - expected_tool_calls=[ - ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}), - ], ), id="cmd3-single_tool_call", ), @@ -89,9 +78,6 @@ REASONING_CASES = [ {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}} ] <|END_ACTION|>""", - expected_tool_calls=[ - ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}), - ], ), id="cmd4-single_tool_call", ), @@ -236,29 +222,8 @@ class TestExtractReasoning: assert reasoning == case.expected_reasoning content = "".join(content_parts) if content_parts else None - if case.expected_tool_calls: - assert content is None or content == "" - else: - assert content == case.expected_content - - accumulated: dict[int, dict] = {} - for d in tool_call_deltas: - idx = d["index"] - if idx not in accumulated: - accumulated[idx] = {"id": "", "name": "", "arguments": ""} - if d["id"]: - accumulated[idx]["id"] = d["id"] - if d["name"]: - accumulated[idx]["name"] = d["name"] - if d["arguments"]: - accumulated[idx]["arguments"] += d["arguments"] - - assert len(accumulated) == len(case.expected_tool_calls) - for i, expected_tc in enumerate(case.expected_tool_calls): - tc = accumulated[i] - assert tc["id"] == expected_tc.id - assert tc["name"] == expected_tc.name - assert json.loads(tc["arguments"]) == expected_tc.arguments + assert content == case.expected_content + assert tool_call_deltas == [] class TestIsReasoningEnd: diff --git a/tests/tool_parsers/test_cohere_command_tool_parser.py b/tests/tool_parsers/test_cohere_command_tool_parser.py new file mode 100644 index 0000000000..f102474b62 --- /dev/null +++ b/tests/tool_parsers/test_cohere_command_tool_parser.py @@ -0,0 +1,510 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any + +import pytest + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.tool_parsers.cohere_command_tool_parser import ( + CohereCommand3ToolParser, + CohereCommand4ToolParser, +) + + +@dataclass +class ExpectedToolCall: + id: str + name: str + arguments: dict + + +@dataclass +class ToolCallCase: + parser_cls: Any + model_output: str + expected_tool_calls: list[ExpectedToolCall] = field(default_factory=list) + expected_reasoning: str | None = None + expected_content: str | None = None + + +TOOL_CALL_CASES = [ + pytest.param( + ToolCallCase( + parser_cls=CohereCommand3ToolParser, + model_output="""\ +<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|> +[ + {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}} +] +<|END_ACTION|>""", + expected_tool_calls=[ + ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}), + ], + expected_reasoning="i will call foo with query1", + ), + id="cmd3-single_tool_call", + ), + pytest.param( + ToolCallCase( + parser_cls=CohereCommand4ToolParser, + model_output="""\ +<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|> +[ + {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}} +] +<|END_ACTION|>""", + expected_tool_calls=[ + ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}), + ], + expected_reasoning="i will call foo with query1", + ), + id="cmd4-single_tool_call", + ), + pytest.param( + ToolCallCase( + parser_cls=CohereCommand3ToolParser, + model_output="""\ +<|START_THINKING|>This is a rainbow emoji: 🌈<|END_THINKING|> +<|START_RESPONSE|>foo bar<|END_RESPONSE|>""", + expected_reasoning="This is a rainbow emoji: 🌈", + expected_content="foo bar", + ), + id="cmd3-citations_no_tool_calls", + ), + pytest.param( + ToolCallCase( + parser_cls=CohereCommand4ToolParser, + model_output="""\ +<|START_THINKING|>This is a rainbow emoji: 🌈<|END_THINKING|> +<|START_RESPONSE|>foo bar<|END_RESPONSE|>""", + expected_reasoning="This is a rainbow emoji: 🌈", + expected_content="foo bar", + ), + id="cmd4-citations_no_tool_calls", + ), + pytest.param( + ToolCallCase( + parser_cls=CohereCommand3ToolParser, + model_output="""\ +<|START_THINKING|>first I think about foo<|END_THINKING|><|START_ACTION|> +[ + {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}, + {"tool_call_id": "1", "tool_name": "bar", "parameters": {"x": 42}} +] +<|END_ACTION|>""", + expected_tool_calls=[ + ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}), + ExpectedToolCall(id="1", name="bar", arguments={"x": 42}), + ], + expected_reasoning="first I think about foo", + ), + id="cmd3-multiple_tool_calls", + ), + pytest.param( + ToolCallCase( + parser_cls=CohereCommand4ToolParser, + model_output="""\ +<|START_THINKING|>first I think about foo<|END_THINKING|><|START_ACTION|> +[ + {"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}, + {"tool_call_id": "1", "tool_name": "bar", "parameters": {"x": 42}} +] +<|END_ACTION|>""", + expected_tool_calls=[ + ExpectedToolCall(id="0", name="foo", arguments={"query": "query1"}), + ExpectedToolCall(id="1", name="bar", arguments={"x": 42}), + ], + expected_reasoning="first I think about foo", + ), + id="cmd4-multiple_tool_calls", + ), + pytest.param( + ToolCallCase( + parser_cls=CohereCommand3ToolParser, + model_output="""\ +<|START_THINKING|>just think, no response<|END_THINKING|>""", + expected_reasoning="just think, no response", + ), + id="cmd3-reasoning_only", + ), + pytest.param( + ToolCallCase( + parser_cls=CohereCommand4ToolParser, + model_output="""\ +<|START_THINKING|>just think, no response<|END_THINKING|>""", + expected_reasoning="just think, no response", + ), + id="cmd4-reasoning_only", + ), +] + + +class MockCohereTokenizer: + """Minimal byte-level stand-in for the Cohere tokenizer.""" + + def get_vocab(self) -> dict[str, int]: + return {} + + def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: + return list(text.encode("utf-8")) + + def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str: + return bytes(ids).decode("utf-8", errors="replace") + + +@pytest.fixture(scope="module") +def tokenizer() -> MockCohereTokenizer: + return MockCohereTokenizer() + + +@pytest.fixture +def request_obj() -> ChatCompletionRequest: + return ChatCompletionRequest(messages=[], model="test-model") + + +REPLACEMENT_CHAR = "\ufffd" + + +def _token_deltas(tokenizer: MockCohereTokenizer, text: str) -> list[str]: + """Decode per-token string deltas, buffering incomplete multi-byte chars.""" + ids = tokenizer.encode(text, add_special_tokens=False) + deltas: list[str] = [] + prev = "" + for i in range(1, len(ids) + 1): + current = tokenizer.decode(ids[:i], skip_special_tokens=False) + if current.endswith(REPLACEMENT_CHAR): + continue + delta = current[len(prev) :] + if delta: + deltas.append(delta) + prev = current + return deltas + + +@dataclass +class StreamingResult: + tool_calls: dict[int, dict] + reasoning: str | None + content: str | None + + +def _run_streaming_over_deltas( + parser, + deltas: list[str], + request_obj: ChatCompletionRequest, +) -> StreamingResult: + accumulated: dict[int, dict] = {} + reasoning_parts: list[str] = [] + content_parts: list[str] = [] + previous_text = "" + + for token_str in deltas: + current_text = previous_text + token_str + delta = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=token_str, + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=request_obj, + ) + if delta is not None: + if delta.reasoning is not None: + reasoning_parts.append(delta.reasoning) + if delta.content is not None: + content_parts.append(delta.content) + for tc in delta.tool_calls: + idx = tc.index + if idx not in accumulated: + accumulated[idx] = {"id": "", "name": "", "arguments": ""} + if tc.id: + accumulated[idx]["id"] = tc.id + if tc.function and tc.function.name: + accumulated[idx]["name"] = tc.function.name + if tc.function and tc.function.arguments: + accumulated[idx]["arguments"] += tc.function.arguments + previous_text = current_text + + return StreamingResult( + tool_calls=accumulated, + reasoning="".join(reasoning_parts) if reasoning_parts else None, + content="".join(content_parts) if content_parts else None, + ) + + +def _run_streaming( + parser, + tokenizer: MockCohereTokenizer, + model_output: str, + request_obj: ChatCompletionRequest, +) -> StreamingResult: + return _run_streaming_over_deltas( + parser, + _token_deltas(tokenizer, model_output), + request_obj, + ) + + +@pytest.mark.parametrize("case", TOOL_CALL_CASES) +class TestExtractToolCalls: + def test_streaming( + self, + tokenizer: MockCohereTokenizer, + request_obj: ChatCompletionRequest, + case: ToolCallCase, + ): + parser = case.parser_cls(tokenizer) + streamed = _run_streaming(parser, tokenizer, case.model_output, request_obj) + + assert len(streamed.tool_calls) == len(case.expected_tool_calls) + + for i, expected_tc in enumerate(case.expected_tool_calls): + tc = streamed.tool_calls[i] + assert tc["id"] == expected_tc.id + assert tc["name"] == expected_tc.name + assert json.loads(tc["arguments"]) == expected_tc.arguments + + def test_streaming_reasoning( + self, + tokenizer: MockCohereTokenizer, + request_obj: ChatCompletionRequest, + case: ToolCallCase, + ): + parser = case.parser_cls(tokenizer) + streamed = _run_streaming(parser, tokenizer, case.model_output, request_obj) + + assert streamed.reasoning == case.expected_reasoning + + def test_streaming_content( + self, + tokenizer: MockCohereTokenizer, + request_obj: ChatCompletionRequest, + case: ToolCallCase, + ): + parser = case.parser_cls(tokenizer) + streamed = _run_streaming(parser, tokenizer, case.model_output, request_obj) + + assert streamed.content == case.expected_content + + def test_nonstreaming( + self, + request_obj: ChatCompletionRequest, + tokenizer: MockCohereTokenizer, + case: ToolCallCase, + ): + parser = case.parser_cls(tokenizer) + result = parser.extract_tool_calls(case.model_output, request_obj) + + assert result.tools_called == (len(case.expected_tool_calls) > 0) + assert len(result.tool_calls) == len(case.expected_tool_calls) + + for actual_tc, expected_tc in zip(result.tool_calls, case.expected_tool_calls): + assert actual_tc.type == "function" + assert actual_tc.function.name == expected_tc.name + assert json.loads(actual_tc.function.arguments) == expected_tc.arguments + + def test_streaming_nonstreaming_agree( + self, + tokenizer: MockCohereTokenizer, + request_obj: ChatCompletionRequest, + case: ToolCallCase, + ): + parser_streaming = case.parser_cls(tokenizer) + parser_nonstreaming = case.parser_cls(tokenizer) + + streamed = _run_streaming( + parser_streaming, + tokenizer, + case.model_output, + request_obj, + ) + result = parser_nonstreaming.extract_tool_calls( + case.model_output, + request_obj, + ) + + assert len(streamed.tool_calls) == len(result.tool_calls) + + for i, actual_tc in enumerate(result.tool_calls): + assert streamed.tool_calls[i]["name"] == actual_tc.function.name + assert json.loads(streamed.tool_calls[i]["arguments"]) == json.loads( + actual_tc.function.arguments + ) + + +SPECIAL_TOKEN_MARKERS = ( + "<|START_THINKING|>", + "<|END_THINKING|>", + "<|START_RESPONSE|>", + "<|END_RESPONSE|>", + "<|START_ACTION|>", + "<|END_ACTION|>", + "<|START_TEXT|>", + "<|END_TEXT|>", +) + + +def _multi_token_deltas( + tokenizer: MockCohereTokenizer, + text: str, + chunk_size: int, +) -> list[str]: + ids = tokenizer.encode(text, add_special_tokens=False) + deltas: list[str] = [] + prev = "" + i = 0 + while i < len(ids): + end = min(len(ids), i + chunk_size) + current = tokenizer.decode(ids[:end], skip_special_tokens=False) + i = end + if current.endswith(REPLACEMENT_CHAR): + continue + delta = current[len(prev) :] + if delta: + deltas.append(delta) + prev = current + return deltas + + +class TestSpeculativeDecodingMultiTokenDelta: + MODEL_OUTPUT = ( + "<|START_THINKING|> i will call foo with query1<|END_THINKING|>" + "<|START_ACTION|>\n" + '[\n {"tool_call_id": "0", "tool_name": "foo", ' + '"parameters": {"query": "query1"}}\n]\n' + "<|END_ACTION|>" + ) + + @pytest.mark.parametrize( + "parser_cls", + [CohereCommand3ToolParser, CohereCommand4ToolParser], + ids=["cmd3", "cmd4"], + ) + @pytest.mark.parametrize("chunk_size", [2, 3, 4, 6]) + def test_no_special_token_leak_in_streaming_deltas( + self, + tokenizer: MockCohereTokenizer, + request_obj: ChatCompletionRequest, + parser_cls, + chunk_size: int, + ): + parser = parser_cls(tokenizer) + chunked = _multi_token_deltas(tokenizer, self.MODEL_OUTPUT, chunk_size) + + previous_text = "" + for token_str in chunked: + current_text = previous_text + token_str + delta = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=token_str, + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=request_obj, + ) + previous_text = current_text + if delta is None: + continue + + fields: list[tuple[str, str | None]] = [ + ("reasoning", delta.reasoning), + ("content", delta.content), + ] + for tc in delta.tool_calls or []: + if tc.function: + fields.append(("tool_call.name", tc.function.name)) + fields.append(("tool_call.arguments", tc.function.arguments)) + + for marker in SPECIAL_TOKEN_MARKERS: + for field_name, value in fields: + assert value is None or marker not in value, ( + f"special token {marker!r} leaked into {field_name} " + f"with chunk_size={chunk_size} delta={delta!r}" + ) + + @pytest.mark.parametrize( + "parser_cls", + [CohereCommand3ToolParser, CohereCommand4ToolParser], + ids=["cmd3", "cmd4"], + ) + @pytest.mark.parametrize("chunk_size", [2, 3, 4, 6]) + def test_multi_token_chunks_still_produce_correct_tool_call( + self, + tokenizer: MockCohereTokenizer, + request_obj: ChatCompletionRequest, + parser_cls, + chunk_size: int, + ): + parser = parser_cls(tokenizer) + chunked = _multi_token_deltas(tokenizer, self.MODEL_OUTPUT, chunk_size) + streamed = _run_streaming_over_deltas(parser, chunked, request_obj) + + assert len(streamed.tool_calls) == 1 + tc = streamed.tool_calls[0] + assert tc["id"] == "0" + assert tc["name"] == "foo" + assert json.loads(tc["arguments"]) == {"query": "query1"} + + +class TestStreamingDeltaShape: + @pytest.mark.parametrize( + "parser_cls", + [CohereCommand3ToolParser, CohereCommand4ToolParser], + ids=["cmd3", "cmd4"], + ) + def test_reasoning_and_tool_calls_are_separate_deltas( + self, + tokenizer: MockCohereTokenizer, + request_obj: ChatCompletionRequest, + parser_cls, + ): + parser = parser_cls(tokenizer) + model_output = ( + "<|START_THINKING|> i will call foo with query1<|END_THINKING|>" + "<|START_ACTION|>\n" + '[\n {"tool_call_id": "0", "tool_name": "foo", ' + '"parameters": {"query": "query1"}}\n]\n' + "<|END_ACTION|>" + ) + + token_strings = _token_deltas(tokenizer, model_output) + previous_text = "" + saw_reasoning = False + saw_tool_call = False + + for token_str in token_strings: + current_text = previous_text + token_str + delta = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=token_str, + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=request_obj, + ) + if delta is not None: + populated = [ + delta.content is not None, + delta.reasoning is not None, + bool(delta.tool_calls), + ] + assert sum(populated) == 1, ( + "A single streaming delta must carry exactly one of " + f"content/reasoning/tool_calls, got {delta!r}" + ) + if delta.reasoning is not None: + saw_reasoning = True + if delta.tool_calls: + saw_tool_call = True + previous_text = current_text + + assert saw_reasoning, "expected at least one reasoning delta" + assert saw_tool_call, "expected at least one tool-call delta" diff --git a/vllm/reasoning/cohere_command_reasoning_parser.py b/vllm/reasoning/cohere_command_reasoning_parser.py index 340f7e022c..dffea49731 100644 --- a/vllm/reasoning/cohere_command_reasoning_parser.py +++ b/vllm/reasoning/cohere_command_reasoning_parser.py @@ -16,7 +16,7 @@ except ImportError as e: raise ImportError( "The Cohere reasoning parser requires the `cohere_melody` " "package, which is not installed. Install it with:\n" - " pip install cohere_melody" + " pip install 'cohere-melody>=0.11.1'" ) from e @@ -132,7 +132,7 @@ def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str: tool_dictionary[tool_name] = f"{tool_name} ::= {tool_name}root\n{tool_grammar}" # Emitted grammar shape: # root ::= tools - # tools ::= ws "[" ws tool ws ("," ws tool)* ws "]" ws + # tools ::= ws "[" ws tool (ws "," ws tool)* ws "]" ws # ws ::= (" " | "\t" | "\n")* # tool ::= | | ... (one alternative per input) # ::= root (per-tool xgrammar rules) @@ -140,7 +140,7 @@ def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str: tool_alternatives = "tool ::= " + " | ".join(tool_dictionary.keys()) tool_rules = "\n ".join(tool_dictionary.values()) grammar = f"""root ::= tools - tools ::= ws "[" ws tool ws ("," ws tool)* ws "]" ws + tools ::= ws "[" ws tool (ws "," ws tool)* ws "]" ws ws ::= (" " | "\\t" | "\\n")* {tool_alternatives} {tool_rules} @@ -699,7 +699,9 @@ class CohereCommand3ReasoningParser(BaseCohereCommandReasoningParser): super().__init__( tokenizer, *args, - streaming_opts=PyFilterOptions().cmd3().stream_non_grounded_answer(), + streaming_opts=( + PyFilterOptions().cmd3().no_tools().stream_non_grounded_answer() + ), unary_opts=PyFilterOptions().cmd3().no_tools(), **kwargs, ) @@ -710,7 +712,9 @@ class CohereCommand4ReasoningParser(BaseCohereCommandReasoningParser): super().__init__( tokenizer, *args, - streaming_opts=PyFilterOptions().cmd4().stream_non_grounded_answer(), + streaming_opts=( + PyFilterOptions().cmd4().no_tools().stream_non_grounded_answer() + ), unary_opts=PyFilterOptions().cmd4().no_tools(), **kwargs, ) diff --git a/vllm/renderers/cohere.py b/vllm/renderers/cohere.py index dbe5bf0ae5..d3bf9dbfd8 100644 --- a/vllm/renderers/cohere.py +++ b/vllm/renderers/cohere.py @@ -209,7 +209,7 @@ def _try_import_melody(): raise ImportError( "The `cohere` tokenizer/renderer mode requires the " "`cohere_melody` package. Install it via " - "`pip install cohere-melody` or build from " + "`pip install 'cohere-melody>=0.11.1'` or build from " "https://github.com/cohere-ai/melody." ) from e diff --git a/vllm/tool_parsers/cohere_command_tool_parser.py b/vllm/tool_parsers/cohere_command_tool_parser.py index 6ce753b993..558d71ccc5 100644 --- a/vllm/tool_parsers/cohere_command_tool_parser.py +++ b/vllm/tool_parsers/cohere_command_tool_parser.py @@ -9,7 +9,7 @@ except ImportError as e: raise ImportError( "The Cohere tool parser requires the `cohere_melody` " "package, which is not installed. Install it with:\n" - " pip install cohere_melody" + " pip install 'cohere-melody>=0.11.1'" ) from e from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -130,8 +130,8 @@ class CohereCommand3ToolParser(BaseCohereCommandToolParser): ): super().__init__( tokenizer, - streaming_opts=PyFilterOptions().cmd3(), - unary_opts=PyFilterOptions().cmd3(), + streaming_opts=PyFilterOptions().cmd3().start_in_answer(), + unary_opts=PyFilterOptions().cmd3().start_in_answer(), ) @@ -143,6 +143,6 @@ class CohereCommand4ToolParser(BaseCohereCommandToolParser): ): super().__init__( tokenizer, - streaming_opts=PyFilterOptions().cmd4(), - unary_opts=PyFilterOptions().cmd4(), + streaming_opts=PyFilterOptions().cmd4().start_in_answer(), + unary_opts=PyFilterOptions().cmd4().start_in_answer(), )