[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,240 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.openai.decisions.adapters import (
|
||||
input_text,
|
||||
make_answer,
|
||||
make_read_question,
|
||||
)
|
||||
from vllm.entrypoints.openai.decisions.protocol import DecisionRequest
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
StructuredDecisionError,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
|
||||
def request(**kwargs):
|
||||
return DecisionRequest.model_validate(
|
||||
{
|
||||
"model": "test-model",
|
||||
"input": "The package is broken.",
|
||||
"questions": [{"type": "predicate", "instructions": "Is it damaged?"}],
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_predicate_returns_probability_of_true_and_null_name():
|
||||
question = request().questions[0]
|
||||
read = make_read_question(0, question)
|
||||
assert [option.name for option in read.options] == ["false", "true"]
|
||||
assert make_answer(question, [0.1, 0.9], 0.5).model_dump() == {
|
||||
"type": "predicate",
|
||||
"name": None,
|
||||
"probability": 0.9,
|
||||
}
|
||||
|
||||
|
||||
def test_choice_preserves_boolean_and_string_values():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "choice",
|
||||
"name": "decision",
|
||||
"instructions": "Select a value.",
|
||||
"choices": [{"value": True}, {"value": "true"}, {"value": False}],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
read = make_read_question(0, question)
|
||||
assert [option.name for option in read.options] == ["true", '"true"', "false"]
|
||||
answer = make_answer(question, [0.7, 0.2, 0.1], 0.5).model_dump()
|
||||
assert answer["name"] == "decision"
|
||||
assert answer["choice"] is True
|
||||
assert answer["probabilities"] == [
|
||||
{"value": True, "probability": 0.7},
|
||||
{"value": "true", "probability": 0.2},
|
||||
{"value": False, "probability": 0.1},
|
||||
]
|
||||
assert answer["confidence"] == pytest.approx(0.35)
|
||||
|
||||
|
||||
def test_score_is_weighted_average_of_zero_based_level_indices():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "Rate severity.",
|
||||
"levels": [{"label": label} for label in ("low", "medium", "high")],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
answer = make_answer(question, [0.1, 0.7, 0.2], 0.8).model_dump()
|
||||
assert answer["score"] == pytest.approx(1.1)
|
||||
assert answer["probabilities"] == [
|
||||
{"value": 0, "label": "low", "probability": 0.1},
|
||||
{"value": 1, "label": "medium", "probability": 0.7},
|
||||
{"value": 2, "label": "high", "probability": 0.2},
|
||||
]
|
||||
assert answer["confidence"] == pytest.approx(0.56)
|
||||
|
||||
|
||||
def test_text_messages_preserve_evidence_order():
|
||||
parsed = request(
|
||||
input=[
|
||||
{"role": "user", "content": "first"},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "second"},
|
||||
{"type": "input_text", "text": "third"},
|
||||
],
|
||||
},
|
||||
]
|
||||
)
|
||||
assert input_text(parsed.input) == "first\n\nsecond\nthird"
|
||||
assert input_text([]) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("evidence", ["", [], [{"role": "user", "content": []}]])
|
||||
def test_empty_input_is_valid_under_the_openai_schema(evidence):
|
||||
assert input_text(request(input=evidence).input) == ""
|
||||
|
||||
|
||||
def test_score_levels_with_the_same_label_keep_their_positions():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "Rate severity.",
|
||||
"levels": [
|
||||
{"label": "damage", "description": "cosmetic"},
|
||||
{"label": "damage", "description": "unusable"},
|
||||
],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
read = make_read_question(0, question)
|
||||
assert read.labels == ("A", "B")
|
||||
answer = make_answer(question, [0.1, 0.9], 0.8).model_dump()
|
||||
assert answer["score"] == pytest.approx(0.9)
|
||||
assert [p["value"] for p in answer["probabilities"]] == [0, 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override,match",
|
||||
[
|
||||
({"input": [{"role": "assistant", "content": "x"}]}, "user"),
|
||||
(
|
||||
{
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,AA==",
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"text input only",
|
||||
),
|
||||
({"input": {"text": "x"}}, "Input should"),
|
||||
({"questions": []}, "at least 1"),
|
||||
(
|
||||
{"questions": [{"type": "predicate", "instructions": "x", "name": None}]},
|
||||
"name must be a string",
|
||||
),
|
||||
({"questions": [{"type": "predicate", "instructions": 1}]}, "valid string"),
|
||||
(
|
||||
{"questions": [{"type": "predicate", "instructions": "x", "criteria": {}}]},
|
||||
"Extra inputs",
|
||||
),
|
||||
({"safety_identifier": "x" * 129}, "at most 128"),
|
||||
({"stream": True}, "Extra inputs"),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_or_unsupported_inputs(override, match):
|
||||
with pytest.raises(ValidationError, match=match):
|
||||
request(**override)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("values", [[True, True], ["a", "a"], [1, "a"], [None, "a"]])
|
||||
def test_choice_rejects_duplicates_and_non_string_non_boolean_values(values):
|
||||
with pytest.raises(ValidationError):
|
||||
request(
|
||||
questions=[
|
||||
{
|
||||
"type": "choice",
|
||||
"instructions": "x",
|
||||
"choices": [{"value": v} for v in values],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"count,valid", [(1, False), (2, True), (255, True), (256, False)]
|
||||
)
|
||||
def test_choice_limits(count, valid):
|
||||
data = {
|
||||
"questions": [
|
||||
{
|
||||
"type": "choice",
|
||||
"instructions": "x",
|
||||
"choices": [{"value": str(i)} for i in range(count)],
|
||||
}
|
||||
]
|
||||
}
|
||||
if valid:
|
||||
assert len(request(**data).questions[0].choices) == count
|
||||
else:
|
||||
with pytest.raises(ValidationError):
|
||||
request(**data)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"count,valid", [(1, False), (2, True), (10, True), (11, False)]
|
||||
)
|
||||
def test_score_limits(count, valid):
|
||||
data = {
|
||||
"questions": [
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "x",
|
||||
"levels": [{"label": str(i)} for i in range(count)],
|
||||
}
|
||||
]
|
||||
}
|
||||
if valid:
|
||||
request(**data)
|
||||
else:
|
||||
with pytest.raises(ValidationError):
|
||||
request(**data)
|
||||
|
||||
|
||||
def test_question_limit_and_optional_safety_identifier():
|
||||
questions = [{"type": "predicate", "instructions": "x"}] * 200
|
||||
assert len(request(questions=questions, safety_identifier=None).questions) == 200
|
||||
with pytest.raises(ValidationError, match="at most 200"):
|
||||
request(questions=[*questions, questions[0]])
|
||||
|
||||
|
||||
def test_backend_rejects_more_choices_than_single_token_labels():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "choice",
|
||||
"instructions": "x",
|
||||
"choices": [{"value": str(i)} for i in range(27)],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
with pytest.raises(StructuredDecisionError, match="at most 26 choices"):
|
||||
make_read_question(0, question)
|
||||
@@ -1,179 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
QUESTION_TYPES,
|
||||
Option,
|
||||
Question,
|
||||
QuestionType,
|
||||
StructuredDecisionError,
|
||||
build_question,
|
||||
register_question_type,
|
||||
)
|
||||
|
||||
|
||||
def choice(qid="bucket", criteria=None, instructions="Which team?", max_options=128):
|
||||
return build_question(
|
||||
qid,
|
||||
"choice",
|
||||
instructions,
|
||||
criteria or {"billing": "money", "outage": None, "other": None},
|
||||
max_options,
|
||||
)
|
||||
|
||||
|
||||
def noul(qid="urgent", criteria=None, instructions="Reply within the hour?"):
|
||||
return build_question(qid, "noul", instructions, criteria, 128)
|
||||
|
||||
|
||||
def score(qid="tone", criteria=None, instructions="How angry?", max_options=128):
|
||||
return build_question(
|
||||
qid,
|
||||
"score",
|
||||
instructions,
|
||||
criteria or ["calm", "annoyed", "furious"],
|
||||
max_options,
|
||||
)
|
||||
|
||||
|
||||
def test_choice_labels_follow_option_order():
|
||||
q = choice()
|
||||
assert q.labels == ("A", "B", "C")
|
||||
assert [a.name for a in q.options] == ["billing", "outage", "other"]
|
||||
assert q.options[0].description == "money"
|
||||
wide = choice(criteria={str(i): None for i in range(26)})
|
||||
assert wide.labels[-1] == "Z"
|
||||
|
||||
|
||||
def test_choice_prompt():
|
||||
q = choice()
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: Which team?\n"
|
||||
"A: billing - money\n"
|
||||
"B: outage\n"
|
||||
"C: other\n"
|
||||
"Answer with the letter of one option only."
|
||||
)
|
||||
|
||||
|
||||
def test_choice_answer_shape():
|
||||
q = choice()
|
||||
answer = q.type.answer(q, [0.2, 0.7, 0.1], 0.5)
|
||||
assert answer == {
|
||||
"type": "choice",
|
||||
"choice": "outage",
|
||||
"probabilities": {"billing": 0.2, "outage": 0.7, "other": 0.1},
|
||||
"confidence": 0.35,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"qid,type_name,criteria,match",
|
||||
[
|
||||
("", "choice", {"x": None, "y": None}, "non-empty"),
|
||||
("q", "nope", {"x": None, "y": None}, "unknown question type"),
|
||||
("q", "choice", ["x", "y"], "must map option names"),
|
||||
("q", "noul", ["yes", "no"], "must be an object"),
|
||||
("q", "noul", {"maybe": "same day"}, "must be an object"),
|
||||
("q", "score", {"0": "calm", "1": "furious"}, "ordered list of levels"),
|
||||
("q", "score", ["calm"], "ordered list of levels"),
|
||||
],
|
||||
)
|
||||
def test_build_question_rejects(qid, type_name, criteria, match):
|
||||
with pytest.raises(StructuredDecisionError, match=match):
|
||||
build_question(qid, type_name, "", criteria, 128)
|
||||
|
||||
|
||||
def test_one_option_choice():
|
||||
q = choice(criteria={"only": None})
|
||||
assert q.labels == ("A",)
|
||||
assert q.type.answer(q, [1.0], 0.4)["confidence"] == 0.4
|
||||
|
||||
|
||||
def test_option_limit():
|
||||
with pytest.raises(StructuredDecisionError, match="at most 2 options"):
|
||||
choice(criteria={"x": None, "y": None, "z": None}, max_options=2)
|
||||
with pytest.raises(StructuredDecisionError, match="at most 26 options"):
|
||||
choice(criteria={str(i): None for i in range(27)})
|
||||
|
||||
|
||||
def test_noul_labels_and_prompt():
|
||||
q = noul()
|
||||
assert q.labels == ("yes", "no")
|
||||
assert [a.name for a in q.options] == ["yes", "no"]
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: Reply within the hour?\nyes\nno\nAnswer with yes or no only."
|
||||
)
|
||||
described = noul(criteria={"true": "same day", "false": "later"})
|
||||
assert described.type.prompt(described) == (
|
||||
"Question: Reply within the hour?\n"
|
||||
"yes: same day\n"
|
||||
"no: later\n"
|
||||
"Answer with yes or no only."
|
||||
)
|
||||
|
||||
|
||||
def test_noul_answer_shape():
|
||||
q = noul()
|
||||
answer = q.type.answer(q, [0.8, 0.2], 0.5)
|
||||
assert answer == {
|
||||
"type": "noul",
|
||||
"noul": 0.8,
|
||||
"probabilities": {"yes": 0.8, "no": 0.2},
|
||||
"confidence": 0.4,
|
||||
}
|
||||
|
||||
|
||||
def test_score_labels_are_levels():
|
||||
assert score().labels == ("0", "1", "2")
|
||||
with pytest.raises(StructuredDecisionError, match="at most 10"):
|
||||
score(criteria=[str(i) for i in range(11)])
|
||||
|
||||
|
||||
def test_score_prompt():
|
||||
q = score()
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: How angry?\n"
|
||||
"0: calm\n"
|
||||
"1: annoyed\n"
|
||||
"2: furious\n"
|
||||
"Answer with the number of one level only."
|
||||
)
|
||||
|
||||
|
||||
def test_score_answer_shape():
|
||||
q = score()
|
||||
answer = q.type.answer(q, [0.5, 0.25, 0.25], 0.4)
|
||||
assert answer == {
|
||||
"type": "score",
|
||||
"score": 0.75,
|
||||
"legend": {"0": "calm", "1": "annoyed", "2": "furious"},
|
||||
"probabilities": {"0": 0.5, "1": 0.25, "2": 0.25},
|
||||
"confidence": 0.2,
|
||||
}
|
||||
|
||||
|
||||
def test_registered_type_plugs_in():
|
||||
class BinaryQuestion(QuestionType):
|
||||
name = "test_binary"
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
return [Option("yes"), Option("no")]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
return {"type": self.name, "yes": probs[0]}
|
||||
|
||||
register_question_type(BinaryQuestion)
|
||||
try:
|
||||
q = build_question("ok", "test_binary", "Is it fine?", None, 128)
|
||||
assert q.labels == ("A", "B")
|
||||
assert q.type.answer(q, [0.9, 0.1], 1.0) == {"type": "test_binary", "yes": 0.9}
|
||||
with pytest.raises(ValueError, match="already registered"):
|
||||
register_question_type(BinaryQuestion)
|
||||
finally:
|
||||
del QUESTION_TYPES["test_binary"]
|
||||
@@ -1,260 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--enable-prefix-caching",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def post(server, body):
|
||||
return requests.post(server.url_for("v1/systemone"), json=body)
|
||||
|
||||
|
||||
def choice(instructions, *names):
|
||||
return {
|
||||
"type": "choice",
|
||||
"instructions": instructions,
|
||||
"criteria": dict.fromkeys(names),
|
||||
}
|
||||
|
||||
|
||||
COPY = {
|
||||
"team": choice(
|
||||
"Which team does the message name?", "billing", "shipping", "security"
|
||||
),
|
||||
"lang": choice("Which language does the message name?", "English", "French"),
|
||||
}
|
||||
|
||||
|
||||
def test_choice_decision(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": {"team": "billing", "language": "French"},
|
||||
"questions": COPY,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert body["object"] == "structured_decision"
|
||||
assert list(body["answers"]) == ["team", "lang"]
|
||||
team = body["answers"]["team"]
|
||||
assert team["type"] == "choice"
|
||||
assert set(team["probabilities"]) == {"billing", "shipping", "security"}
|
||||
assert sum(team["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
|
||||
assert team["confidence"] == pytest.approx(
|
||||
max(team["probabilities"].values()) * body["diagnostics"]["team"]["label_mass"]
|
||||
)
|
||||
for diag in body["diagnostics"].values():
|
||||
assert 0.0 < diag["label_mass"] <= 1.0 + 1e-6
|
||||
assert body["usage"]["output_tokens"] == 2
|
||||
assert body["usage"]["input_tokens"] > 0
|
||||
|
||||
|
||||
def test_one_option_choice(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": "Team: billing.",
|
||||
"questions": {
|
||||
"team": choice("Which team does the message name?", "billing")
|
||||
},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
team = body["answers"]["team"]
|
||||
assert team["choice"] == "billing"
|
||||
assert team["probabilities"] == {"billing": 1.0}
|
||||
assert team["confidence"] == pytest.approx(
|
||||
body["diagnostics"]["team"]["label_mass"]
|
||||
)
|
||||
|
||||
|
||||
def test_one_option_coin_flip(server):
|
||||
# A fair coin with only "heads" allowed. The answer is forced, so its
|
||||
# confidence must come from the model's probability of the label.
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": "I flip a fair coin.",
|
||||
"questions": {"coin": choice("How did the coin land?", "heads")},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
coin = body["answers"]["coin"]
|
||||
assert coin["probabilities"] == {"heads": 1.0}
|
||||
assert coin["confidence"] == pytest.approx(
|
||||
body["diagnostics"]["coin"]["label_mass"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state,questions,expected",
|
||||
[
|
||||
(
|
||||
{"team": "billing", "language": "French"},
|
||||
COPY,
|
||||
{"team": "billing", "lang": "French"},
|
||||
),
|
||||
(
|
||||
"My card was charged twice for one order.",
|
||||
{
|
||||
"card": choice("Does the message mention a card?", "yes", "no"),
|
||||
"mood": choice("How does the customer feel?", "happy", "angry"),
|
||||
},
|
||||
{"card": "yes", "mood": "angry"},
|
||||
),
|
||||
(
|
||||
"Hello.",
|
||||
{
|
||||
"sky": choice(
|
||||
"What color is the sky on a clear day?", "blue", "green", "red"
|
||||
)
|
||||
},
|
||||
{"sky": "blue"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_single_read_answers(server, state, questions, expected):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": state,
|
||||
"questions": questions,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
answers = response.json()["answers"]
|
||||
assert {qid: a["choice"] for qid, a in answers.items()} == expected
|
||||
|
||||
|
||||
def test_option_limit(server):
|
||||
body = {
|
||||
"model": MODEL_NAME,
|
||||
"state": "Pick option 7.",
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
ok = post(
|
||||
server, {**body, "questions": {"n": choice("Which?", *map(str, range(26)))}}
|
||||
)
|
||||
assert ok.status_code == 200, ok.text
|
||||
assert len(ok.json()["answers"]["n"]["probabilities"]) == 26
|
||||
over = post(
|
||||
server, {**body, "questions": {"n": choice("Which?", *map(str, range(27)))}}
|
||||
)
|
||||
assert over.status_code == 400
|
||||
assert "at most 26 options" in over.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_thinking_is_refused(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": "x",
|
||||
"questions": {"q": choice("Which?", "a", "b")},
|
||||
"chat_template_kwargs": {"enable_thinking": True},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "thinking must be off" in response.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_noul_and_score(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": {"ticket": "My card was charged twice for one order."},
|
||||
"questions": {
|
||||
"urgent": {"type": "noul", "instructions": "Is money moving?"},
|
||||
"tone": {
|
||||
"type": "score",
|
||||
"instructions": "How upset is the customer?",
|
||||
"criteria": ["calm", "annoyed", "furious"],
|
||||
},
|
||||
},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
urgent = body["answers"]["urgent"]
|
||||
assert urgent["type"] == "noul"
|
||||
assert set(urgent["probabilities"]) == {"yes", "no"}
|
||||
assert urgent["probabilities"]["yes"] + urgent["probabilities"]["no"] == (
|
||||
pytest.approx(1.0, abs=1e-4)
|
||||
)
|
||||
assert urgent["noul"] == urgent["probabilities"]["yes"]
|
||||
assert urgent["confidence"] == pytest.approx(
|
||||
max(urgent["probabilities"].values())
|
||||
* body["diagnostics"]["urgent"]["label_mass"]
|
||||
)
|
||||
tone = body["answers"]["tone"]
|
||||
assert tone["type"] == "score"
|
||||
assert tone["legend"] == {"0": "calm", "1": "annoyed", "2": "furious"}
|
||||
assert set(tone["probabilities"]) == {"0", "1", "2"}
|
||||
assert sum(tone["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
|
||||
assert tone["score"] == pytest.approx(
|
||||
sum(i * p for i, p in enumerate(tone["probabilities"].values()))
|
||||
)
|
||||
assert 0 <= tone["score"] <= 2
|
||||
assert body["usage"]["output_tokens"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"questions,match",
|
||||
[
|
||||
(
|
||||
{
|
||||
"q": {
|
||||
"type": "choice",
|
||||
"criteria": {"a": None, "b": None},
|
||||
"depends_on": ["x"],
|
||||
}
|
||||
},
|
||||
"unknown field(s)",
|
||||
),
|
||||
(
|
||||
{
|
||||
f"q{i}": {"type": "choice", "criteria": {"a": None, "b": None}}
|
||||
for i in range(65)
|
||||
},
|
||||
"at most 64",
|
||||
),
|
||||
({"q": {"type": "noul", "criteria": ["yes", "no"]}}, "must be an object"),
|
||||
({"q": {"type": "score", "criteria": ["calm"]}}, "ordered list of levels"),
|
||||
],
|
||||
)
|
||||
def test_validation_errors(server, questions, match):
|
||||
response = post(server, {"model": MODEL_NAME, "state": "x", "questions": questions})
|
||||
assert response.status_code == 400
|
||||
assert match in response.json()["error"]["message"]
|
||||
Reference in New Issue
Block a user