[Bugfix][Frontend] Apply the stop string limit to Cohere requests (#53750)
Signed-off-by: mhuzaifa3 <mhuzaifa3@outlook.com>
This commit is contained in:
@@ -16,6 +16,7 @@ types plus a few local additions:
|
|||||||
import pytest
|
import pytest
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
from vllm.entrypoints.cohere.protocol import (
|
from vllm.entrypoints.cohere.protocol import (
|
||||||
AssistantMessageResponse,
|
AssistantMessageResponse,
|
||||||
CitationEndEvent,
|
CitationEndEvent,
|
||||||
@@ -94,6 +95,24 @@ class TestCohereChatV2Request:
|
|||||||
tool_choice="ANY", # not REQUIRED/NONE
|
tool_choice="ANY", # not REQUIRED/NONE
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_stop_sequences_bounded_by_env_limit(self):
|
||||||
|
messages = [{"role": "user", "content": "hi"}]
|
||||||
|
limit = envs.VLLM_MAX_STOP_STRINGS
|
||||||
|
|
||||||
|
req = CohereChatV2Request(
|
||||||
|
model="m",
|
||||||
|
messages=messages,
|
||||||
|
stop_sequences=[f"s{i}" for i in range(limit)],
|
||||||
|
)
|
||||||
|
assert len(req.stop_sequences) == limit
|
||||||
|
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
CohereChatV2Request(
|
||||||
|
model="m",
|
||||||
|
messages=messages,
|
||||||
|
stop_sequences=[f"s{i}" for i in range(limit + 1)],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ======================================================================
|
# ======================================================================
|
||||||
# Usage / Logprobs
|
# Usage / Logprobs
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ See https://docs.cohere.com/reference/chat for the upstream spec.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import Any, Literal
|
from typing import Annotated, Any, Literal
|
||||||
|
|
||||||
from cohere import types as _sdk
|
from cohere import types as _sdk
|
||||||
from cohere.types import (
|
from cohere.types import (
|
||||||
@@ -46,6 +46,7 @@ from cohere.types import (
|
|||||||
)
|
)
|
||||||
from pydantic import BaseModel, Field, field_validator
|
from pydantic import BaseModel, Field, field_validator
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
|
|
||||||
# Re-export the SDK wire-format types alongside our local extensions so
|
# Re-export the SDK wire-format types alongside our local extensions so
|
||||||
@@ -152,7 +153,9 @@ class CohereChatV2Request(BaseModel):
|
|||||||
response_format: ResponseFormatV2 | None = None
|
response_format: ResponseFormatV2 | None = None
|
||||||
safety_mode: ChatRequestSafetyMode | None = None
|
safety_mode: ChatRequestSafetyMode | None = None
|
||||||
max_tokens: int | None = None
|
max_tokens: int | None = None
|
||||||
stop_sequences: list[str] | None = None
|
stop_sequences: (
|
||||||
|
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
|
||||||
|
) = None
|
||||||
|
|
||||||
# Sampling
|
# Sampling
|
||||||
temperature: float | None = None
|
temperature: float | None = None
|
||||||
|
|||||||
Reference in New Issue
Block a user