[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,18 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
|
||||
from .utils import MockCohereTokenizer
|
||||
|
||||
|
||||
@pytest.fixture(scope="package")
|
||||
def tokenizer() -> MockCohereTokenizer:
|
||||
return MockCohereTokenizer()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def request_obj() -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(messages=[], model="test-model")
|
||||
@@ -1,288 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tool-call parsing for the unified Cohere Command parser.
|
||||
|
||||
The ``cohere_command3`` / ``cohere_command4`` tool parsers are registry shims;
|
||||
``CohereCommandParser`` (resolved through ``ParserManager``) does the parsing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
|
||||
from .utils import MockCohereTokenizer, drive_parser, make_parser, token_deltas
|
||||
|
||||
SPECIAL_TOKEN_MARKERS = (
|
||||
"<|START_THINKING|>",
|
||||
"<|END_THINKING|>",
|
||||
"<|START_RESPONSE|>",
|
||||
"<|END_RESPONSE|>",
|
||||
"<|START_ACTION|>",
|
||||
"<|END_ACTION|>",
|
||||
"<|START_TEXT|>",
|
||||
"<|END_TEXT|>",
|
||||
)
|
||||
|
||||
DUMMY_TOOLS = [
|
||||
{"type": "function", "function": {"name": name, "parameters": {"type": "object"}}}
|
||||
for name in ("foo", "bar")
|
||||
]
|
||||
|
||||
# (registry name, answer framing tag)
|
||||
VARIANTS = (("cohere_command3", "RESPONSE"), ("cohere_command4", "TEXT"))
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExpectedToolCall:
|
||||
id: str
|
||||
name: str
|
||||
arguments: dict
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallCase:
|
||||
parser_name: str
|
||||
model_output: str
|
||||
expected_tool_calls: list[ExpectedToolCall] = field(default_factory=list)
|
||||
expected_reasoning: str | None = None
|
||||
expected_content: str | None = None
|
||||
|
||||
|
||||
SINGLE_TOOL_CALL = """\
|
||||
<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
|
||||
[
|
||||
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
|
||||
]
|
||||
<|END_ACTION|>"""
|
||||
MULTI_TOOL_CALL = """\
|
||||
<|START_THINKING|>first I think about foo<|END_THINKING|><|START_ACTION|>
|
||||
[
|
||||
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}},
|
||||
{"tool_call_id": "1", "tool_name": "bar", "parameters": {"x": 42}}
|
||||
]
|
||||
<|END_ACTION|>"""
|
||||
CITATIONS_NO_TOOL_CALL = (
|
||||
"<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>\n"
|
||||
"<|START_{tag}|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_{tag}|>"
|
||||
)
|
||||
|
||||
# (case id, model output template, expectations)
|
||||
_CASE_TABLE = (
|
||||
(
|
||||
"single_tool_call",
|
||||
SINGLE_TOOL_CALL,
|
||||
dict(
|
||||
expected_tool_calls=[ExpectedToolCall("0", "foo", {"query": "query1"})],
|
||||
expected_reasoning="i will call foo with query1",
|
||||
),
|
||||
),
|
||||
(
|
||||
"citations_no_tool_calls",
|
||||
CITATIONS_NO_TOOL_CALL,
|
||||
dict(
|
||||
expected_reasoning="This is a rainbow emoji: 🌈", expected_content="foo bar"
|
||||
),
|
||||
),
|
||||
(
|
||||
"multiple_tool_calls",
|
||||
MULTI_TOOL_CALL,
|
||||
dict(
|
||||
expected_tool_calls=[
|
||||
ExpectedToolCall("0", "foo", {"query": "query1"}),
|
||||
ExpectedToolCall("1", "bar", {"x": 42}),
|
||||
],
|
||||
expected_reasoning="first I think about foo",
|
||||
),
|
||||
),
|
||||
(
|
||||
"reasoning_only",
|
||||
"<|START_THINKING|>just think, no response<|END_THINKING|>",
|
||||
dict(expected_reasoning="just think, no response"),
|
||||
),
|
||||
)
|
||||
|
||||
TOOL_CALL_CASES = [
|
||||
pytest.param(
|
||||
ToolCallCase(name, output.replace("{tag}", tag), **expected),
|
||||
id=f"{name}-{case_id}",
|
||||
)
|
||||
for name, tag in VARIANTS
|
||||
for case_id, output, expected in _CASE_TABLE
|
||||
]
|
||||
|
||||
PARSER_NAMES = pytest.mark.parametrize("parser_name", [name for name, _ in VARIANTS])
|
||||
|
||||
|
||||
def _tool_parser(tokenizer: MockCohereTokenizer, parser_name: str) -> Parser:
|
||||
return make_parser(tokenizer, parser_name, DUMMY_TOOLS)
|
||||
|
||||
|
||||
def _tool_request() -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
messages=[], model="test-model", tools=DUMMY_TOOLS, tool_choice="auto"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StreamingResult:
|
||||
tool_calls: dict[int, dict]
|
||||
reasoning: str | None
|
||||
content: str | None
|
||||
|
||||
|
||||
def run_streaming(
|
||||
tokenizer: MockCohereTokenizer,
|
||||
parser_name: str,
|
||||
model_output: str,
|
||||
chunk_size: int = 1,
|
||||
) -> StreamingResult:
|
||||
deltas = drive_parser(
|
||||
_tool_parser(tokenizer, parser_name),
|
||||
_tool_request(),
|
||||
token_deltas(tokenizer, model_output, chunk_size),
|
||||
)
|
||||
accumulated: dict[int, dict] = {}
|
||||
for delta in deltas:
|
||||
for tc in delta.tool_calls or []:
|
||||
slot = accumulated.setdefault(
|
||||
tc.index, {"id": "", "name": "", "arguments": ""}
|
||||
)
|
||||
if tc.id:
|
||||
slot["id"] = tc.id
|
||||
if tc.function and tc.function.name:
|
||||
slot["name"] = tc.function.name
|
||||
if tc.function and tc.function.arguments:
|
||||
slot["arguments"] += tc.function.arguments
|
||||
return StreamingResult(
|
||||
tool_calls=accumulated,
|
||||
reasoning="".join(d.reasoning for d in deltas if d.reasoning) or None,
|
||||
content="".join(d.content for d in deltas if d.content) or None,
|
||||
)
|
||||
|
||||
|
||||
def run_nonstreaming(
|
||||
tokenizer: MockCohereTokenizer, parser_name: str, model_output: str
|
||||
) -> tuple[str | None, str | None, list]:
|
||||
reasoning, content, tool_calls = _tool_parser(tokenizer, parser_name).parse(
|
||||
model_output, _tool_request(), enable_auto_tools=True
|
||||
)
|
||||
return reasoning, content, tool_calls or []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", TOOL_CALL_CASES)
|
||||
class TestExtractToolCalls:
|
||||
def test_streaming(self, tokenizer: MockCohereTokenizer, case: ToolCallCase):
|
||||
streamed = run_streaming(tokenizer, case.parser_name, case.model_output)
|
||||
assert len(streamed.tool_calls) == len(case.expected_tool_calls)
|
||||
for i, expected in enumerate(case.expected_tool_calls):
|
||||
tc = streamed.tool_calls[i]
|
||||
assert tc["id"] == expected.id
|
||||
assert tc["name"] == expected.name
|
||||
assert json.loads(tc["arguments"]) == expected.arguments
|
||||
|
||||
def test_streaming_reasoning(
|
||||
self, tokenizer: MockCohereTokenizer, case: ToolCallCase
|
||||
):
|
||||
streamed = run_streaming(tokenizer, case.parser_name, case.model_output)
|
||||
assert streamed.reasoning == case.expected_reasoning
|
||||
|
||||
def test_streaming_content(
|
||||
self, tokenizer: MockCohereTokenizer, case: ToolCallCase
|
||||
):
|
||||
streamed = run_streaming(tokenizer, case.parser_name, case.model_output)
|
||||
assert streamed.content == case.expected_content
|
||||
|
||||
def test_nonstreaming(self, tokenizer: MockCohereTokenizer, case: ToolCallCase):
|
||||
reasoning, content, tool_calls = run_nonstreaming(
|
||||
tokenizer, case.parser_name, case.model_output
|
||||
)
|
||||
assert reasoning == case.expected_reasoning
|
||||
assert content == case.expected_content
|
||||
assert len(tool_calls) == len(case.expected_tool_calls)
|
||||
for actual, expected in zip(tool_calls, case.expected_tool_calls):
|
||||
assert actual.name == expected.name
|
||||
assert json.loads(actual.arguments) == expected.arguments
|
||||
|
||||
def test_streaming_nonstreaming_agree(
|
||||
self, tokenizer: MockCohereTokenizer, case: ToolCallCase
|
||||
):
|
||||
streamed = run_streaming(tokenizer, case.parser_name, case.model_output)
|
||||
_, _, tool_calls = run_nonstreaming(
|
||||
tokenizer, case.parser_name, case.model_output
|
||||
)
|
||||
assert len(streamed.tool_calls) == len(tool_calls)
|
||||
for i, actual in enumerate(tool_calls):
|
||||
assert streamed.tool_calls[i]["name"] == actual.name
|
||||
assert json.loads(streamed.tool_calls[i]["arguments"]) == json.loads(
|
||||
actual.arguments
|
||||
)
|
||||
|
||||
|
||||
class TestSpeculativeDecodingMultiTokenDelta:
|
||||
@PARSER_NAMES
|
||||
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 6])
|
||||
def test_no_special_token_leak_in_streaming_deltas(
|
||||
self, tokenizer: MockCohereTokenizer, parser_name: str, chunk_size: int
|
||||
):
|
||||
deltas = drive_parser(
|
||||
_tool_parser(tokenizer, parser_name),
|
||||
_tool_request(),
|
||||
token_deltas(tokenizer, SINGLE_TOOL_CALL, chunk_size),
|
||||
)
|
||||
for delta in deltas:
|
||||
fields: list[tuple[str, str | None]] = [
|
||||
("reasoning", delta.reasoning),
|
||||
("content", delta.content),
|
||||
]
|
||||
for tc in delta.tool_calls or []:
|
||||
if tc.function:
|
||||
fields.append(("tool_call.name", tc.function.name))
|
||||
fields.append(("tool_call.arguments", tc.function.arguments))
|
||||
for marker in SPECIAL_TOKEN_MARKERS:
|
||||
for field_name, value in fields:
|
||||
assert value is None or marker not in value, (
|
||||
f"special token {marker!r} leaked into {field_name} "
|
||||
f"with chunk_size={chunk_size} delta={delta!r}"
|
||||
)
|
||||
|
||||
@PARSER_NAMES
|
||||
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 6])
|
||||
def test_multi_token_chunks_still_produce_correct_tool_call(
|
||||
self, tokenizer: MockCohereTokenizer, parser_name: str, chunk_size: int
|
||||
):
|
||||
streamed = run_streaming(tokenizer, parser_name, SINGLE_TOOL_CALL, chunk_size)
|
||||
assert len(streamed.tool_calls) == 1
|
||||
tc = streamed.tool_calls[0]
|
||||
assert tc["id"] == "0"
|
||||
assert tc["name"] == "foo"
|
||||
assert json.loads(tc["arguments"]) == {"query": "query1"}
|
||||
|
||||
|
||||
class TestStreamingDeltaShape:
|
||||
@PARSER_NAMES
|
||||
def test_reasoning_and_tool_calls_are_separate_deltas(
|
||||
self, tokenizer: MockCohereTokenizer, parser_name: str
|
||||
):
|
||||
deltas = drive_parser(
|
||||
_tool_parser(tokenizer, parser_name),
|
||||
_tool_request(),
|
||||
token_deltas(tokenizer, SINGLE_TOOL_CALL),
|
||||
)
|
||||
for delta in deltas:
|
||||
populated = [
|
||||
delta.content is not None,
|
||||
delta.reasoning is not None,
|
||||
bool(delta.tool_calls),
|
||||
]
|
||||
assert sum(populated) == 1, (
|
||||
"A single streaming delta must carry exactly one of "
|
||||
f"content/reasoning/tool_calls, got {delta!r}"
|
||||
)
|
||||
assert any(d.reasoning is not None for d in deltas)
|
||||
assert any(d.tool_calls for d in deltas)
|
||||
@@ -1,131 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for the unified Cohere Command parser tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
|
||||
_SPECIAL_TOKEN_RE = re.compile(r"(<\|[A-Z_]+\|>)")
|
||||
REPLACEMENT_CHAR = "�"
|
||||
|
||||
|
||||
class MockCohereTokenizer:
|
||||
"""Byte-level stand-in for the Cohere tokenizer.
|
||||
|
||||
Text round-trips through UTF-8 bytes so a multi-byte character can split
|
||||
across "tokens" (reproducing trailing U+FFFD buffering), while each
|
||||
``<|...|>`` marker is a single id so token-id reasoning-end gating behaves
|
||||
as with the real tokenizer.
|
||||
"""
|
||||
|
||||
_SPECIAL_TOKEN_IDS = {
|
||||
tok: 256 + i
|
||||
for i, tok in enumerate(
|
||||
(
|
||||
"<|START_THINKING|>",
|
||||
"<|END_THINKING|>",
|
||||
"<|CHATBOT_TOKEN|>",
|
||||
"<|START_RESPONSE|>",
|
||||
"<|END_RESPONSE|>",
|
||||
"<|START_TEXT|>",
|
||||
"<|END_TEXT|>",
|
||||
"<|START_ACTION|>",
|
||||
"<|END_ACTION|>",
|
||||
)
|
||||
)
|
||||
}
|
||||
_ID_TO_SPECIAL_TOKEN = {v: k for k, v in _SPECIAL_TOKEN_IDS.items()}
|
||||
|
||||
def convert_tokens_to_ids(self, token: str) -> int:
|
||||
return self._SPECIAL_TOKEN_IDS.get(token, 0)
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
|
||||
ids: list[int] = []
|
||||
for part in _SPECIAL_TOKEN_RE.split(text):
|
||||
if part in self._SPECIAL_TOKEN_IDS:
|
||||
ids.append(self._SPECIAL_TOKEN_IDS[part])
|
||||
else:
|
||||
ids.extend(part.encode("utf-8"))
|
||||
return ids
|
||||
|
||||
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
|
||||
out: list[str] = []
|
||||
for special, run in itertools.groupby(
|
||||
ids, self._ID_TO_SPECIAL_TOKEN.__contains__
|
||||
):
|
||||
if not special:
|
||||
out.append(bytes(run).decode("utf-8", errors="replace"))
|
||||
elif not skip_special_tokens:
|
||||
out.extend(self._ID_TO_SPECIAL_TOKEN[i] for i in run)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def token_deltas(
|
||||
tokenizer: MockCohereTokenizer, text: str, chunk_size: int = 1
|
||||
) -> list[tuple[str, list[int]]]:
|
||||
"""Split ``text`` into ``(delta_text, delta_token_ids)`` steps of
|
||||
``chunk_size`` tokens, buffering incomplete multi-byte sequences (trailing
|
||||
U+FFFD) into the next step as real streaming does."""
|
||||
ids = tokenizer.encode(text)
|
||||
deltas: list[tuple[str, list[int]]] = []
|
||||
prev = ""
|
||||
pending: list[int] = []
|
||||
for start in range(0, len(ids), chunk_size):
|
||||
end = start + chunk_size
|
||||
pending.extend(ids[start:end])
|
||||
current = tokenizer.decode(ids[:end])
|
||||
if current.endswith(REPLACEMENT_CHAR) and end < len(ids):
|
||||
continue
|
||||
deltas.append((current[len(prev) :], pending))
|
||||
prev, pending = current, []
|
||||
return deltas
|
||||
|
||||
|
||||
def make_parser(
|
||||
tokenizer: MockCohereTokenizer,
|
||||
name: str,
|
||||
tools: list[dict] | None = None,
|
||||
chat_template_kwargs: dict | None = None,
|
||||
model_config=None,
|
||||
) -> Parser:
|
||||
"""Resolve the unified Cohere parser through ``ParserManager`` by registry
|
||||
name (``cohere_command3`` / ``cohere_command4``)."""
|
||||
cls = ParserManager.get_parser(name, name, enable_auto_tools=True)
|
||||
assert cls is not None
|
||||
return cls(
|
||||
tokenizer,
|
||||
tools,
|
||||
model_config=model_config,
|
||||
chat_template_kwargs=chat_template_kwargs or {},
|
||||
)
|
||||
|
||||
|
||||
def drive_parser(
|
||||
parser: Parser,
|
||||
request: ChatCompletionRequest,
|
||||
deltas: list[tuple[str, list[int]]],
|
||||
) -> list[DeltaMessage]:
|
||||
"""Feed ``deltas`` through ``parse_delta`` (flushing on the last one) and
|
||||
return the non-``None`` messages in order."""
|
||||
out: list[DeltaMessage] = []
|
||||
for i, (text, ids) in enumerate(deltas):
|
||||
delta = parser.parse_delta(text, ids, request, finished=i == len(deltas) - 1)
|
||||
if delta is not None:
|
||||
out.append(delta)
|
||||
return out
|
||||
|
||||
|
||||
def stream_parser(parser, request, tokenizer, text: str, chunk_size: int = 1):
|
||||
"""Stream ``text`` through ``parser`` in ``chunk_size``-token deltas."""
|
||||
return drive_parser(parser, request, token_deltas(tokenizer, text, chunk_size))
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
Reference in New Issue
Block a user