[Frontend] Consolidate scale out entrypoints (#44512)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1,554 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.disagg.protocol import (
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
MODEL_NAME = "openai-community/gpt2"
|
||||
BASE_MODEL_PATHS = [
|
||||
BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockHFConfig:
|
||||
model_type: str = "any"
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockModelConfig:
|
||||
task = "generate"
|
||||
runner_type = "generate"
|
||||
model = MODEL_NAME
|
||||
tokenizer = MODEL_NAME
|
||||
trust_remote_code = False
|
||||
tokenizer_mode = "auto"
|
||||
max_model_len = 100
|
||||
tokenizer_revision = None
|
||||
multimodal_config = MultiModalConfig()
|
||||
hf_config = MockHFConfig()
|
||||
hf_text_config = MockHFConfig()
|
||||
logits_processors: list[str] | None = None
|
||||
diff_sampling_param: dict | None = None
|
||||
allowed_local_media_path: str = ""
|
||||
allowed_media_domains: list[str] | None = None
|
||||
encoder_config = None
|
||||
generation_config: str = "auto"
|
||||
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
skip_tokenizer_init = False
|
||||
is_encoder_decoder: bool = False
|
||||
is_multimodal_model: bool = False
|
||||
renderer_num_workers: int = 1
|
||||
|
||||
def get_diff_sampling_param(self):
|
||||
return self.diff_sampling_param or {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockParallelConfig:
|
||||
_api_process_rank: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockSchedulerConfig:
|
||||
max_num_seqs: int = 128
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockVllmConfig:
|
||||
model_config: MockModelConfig
|
||||
parallel_config: MockParallelConfig
|
||||
scheduler_config: MockSchedulerConfig = field(default_factory=MockSchedulerConfig)
|
||||
|
||||
|
||||
def _build_renderer(model_config: MockModelConfig):
|
||||
return renderer_from_config(
|
||||
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
|
||||
)
|
||||
|
||||
|
||||
def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
|
||||
models = OpenAIServingModels(
|
||||
engine_client=engine,
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
serving = ServingTokens(
|
||||
engine,
|
||||
models,
|
||||
online_renderer=online_renderer,
|
||||
request_logger=None,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def _fake_preprocess(*args, **kwargs):
|
||||
return [{"prompt_token_ids": [1, 2, 3]}]
|
||||
|
||||
serving.online_renderer.preprocess_completion = AsyncMock(
|
||||
side_effect=_fake_preprocess
|
||||
)
|
||||
return serving
|
||||
|
||||
|
||||
def _make_request_output(
|
||||
request_id: str,
|
||||
token_ids: list[int],
|
||||
finish_reason: str | None = None,
|
||||
finished: bool = False,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
logprobs: list[dict[int, Any] | None] | None = None,
|
||||
num_cached_tokens: int | None = None,
|
||||
index: int = 0,
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id=request_id,
|
||||
prompt=None,
|
||||
prompt_token_ids=prompt_token_ids or [1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=index,
|
||||
text="",
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=None,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
],
|
||||
finished=finished,
|
||||
metrics=None,
|
||||
lora_request=None,
|
||||
encoder_prompt=None,
|
||||
encoder_prompt_token_ids=None,
|
||||
num_cached_tokens=num_cached_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _mock_engine() -> MagicMock:
|
||||
engine = MagicMock(spec=AsyncLLM)
|
||||
engine.errored = False
|
||||
engine.model_config = MockModelConfig()
|
||||
engine.vllm_config = MockVllmConfig(
|
||||
engine.model_config, parallel_config=MockParallelConfig()
|
||||
)
|
||||
engine.input_processor = MagicMock()
|
||||
engine.renderer = _build_renderer(engine.model_config)
|
||||
return engine
|
||||
|
||||
|
||||
def _parse_sse_chunks(chunks: list[str]) -> list[Any]:
|
||||
"""Parse SSE chunks into dicts (JSON) or raw strings ([DONE])."""
|
||||
parsed: list[Any] = []
|
||||
for chunk in chunks:
|
||||
assert chunk.startswith("data: ") and chunk.endswith("\n\n")
|
||||
payload = chunk[len("data: ") : -len("\n\n")]
|
||||
if payload == "[DONE]":
|
||||
parsed.append("[DONE]")
|
||||
else:
|
||||
parsed.append(json.loads(payload))
|
||||
return parsed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[10], finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
model=MODEL_NAME,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
|
||||
assert isinstance(response, GenerateResponse)
|
||||
assert (
|
||||
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_basic():
|
||||
"""Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10])
|
||||
yield _make_request_output("req-1", token_ids=[20, 30])
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[40], finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
parsed = _parse_sse_chunks(chunks)
|
||||
|
||||
# 3 data chunks + [DONE]
|
||||
assert parsed[-1] == "[DONE]"
|
||||
data_chunks = [c for c in parsed if c != "[DONE]"]
|
||||
assert len(data_chunks) == 3
|
||||
|
||||
assert data_chunks[0]["choices"][0]["token_ids"] == [10]
|
||||
assert data_chunks[1]["choices"][0]["token_ids"] == [20, 30]
|
||||
assert data_chunks[2]["choices"][0]["token_ids"] == [40]
|
||||
assert data_chunks[2]["choices"][0]["finish_reason"] == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_error_mid_generation():
|
||||
"""finish_reason='error' mid-stream yields error chunk then [DONE]."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10])
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[20], finish_reason="error", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
assert len(chunks) >= 2
|
||||
assert any("Internal server error" in chunk for chunk in chunks), (
|
||||
f"Expected error message in chunks: {chunks}"
|
||||
)
|
||||
assert chunks[-1] == "data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_error_with_empty_delta():
|
||||
"""finish_reason='error' with empty delta_token_ids still raises."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10])
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[], finish_reason="error", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
assert any("Internal server error" in chunk for chunk in chunks), (
|
||||
f"Expected error message in chunks: {chunks}"
|
||||
)
|
||||
assert chunks[-1] == "data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_skips_empty_token_output():
|
||||
"""Outputs with empty token_ids are skipped (no chunk emitted)."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10])
|
||||
yield _make_request_output("req-1", token_ids=[])
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[20], finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
parsed = _parse_sse_chunks(chunks)
|
||||
assert parsed[-1] == "[DONE]"
|
||||
data_chunks = [c for c in parsed if c != "[DONE]"]
|
||||
|
||||
# Only 2 data chunks — the empty one is skipped
|
||||
assert len(data_chunks) == 2
|
||||
assert data_chunks[0]["choices"][0]["token_ids"] == [10]
|
||||
assert data_chunks[1]["choices"][0]["token_ids"] == [20]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_include_usage():
|
||||
"""stream_options.include_usage emits a final usage-only chunk."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10])
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[20], finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
stream_options=StreamOptions(include_usage=True),
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
parsed = _parse_sse_chunks(chunks)
|
||||
assert parsed[-1] == "[DONE]"
|
||||
|
||||
# The chunk before [DONE] should be the usage-only chunk
|
||||
usage_chunk = parsed[-2]
|
||||
assert usage_chunk["choices"] == []
|
||||
assert usage_chunk["usage"]["prompt_tokens"] == 3
|
||||
assert usage_chunk["usage"]["completion_tokens"] == 2
|
||||
assert usage_chunk["usage"]["total_tokens"] == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_continuous_usage():
|
||||
"""continuous_usage_stats adds usage to every data chunk."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output("req-1", token_ids=[10])
|
||||
yield _make_request_output(
|
||||
"req-1", token_ids=[20], finish_reason="stop", finished=True
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
stream_options=StreamOptions(
|
||||
include_usage=True,
|
||||
continuous_usage_stats=True,
|
||||
),
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
parsed = _parse_sse_chunks(chunks)
|
||||
data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
|
||||
|
||||
# Every data chunk should have usage
|
||||
for i, dc in enumerate(data_chunks):
|
||||
assert dc["usage"] is not None, f"chunk {i} missing usage"
|
||||
assert dc["usage"]["prompt_tokens"] == 3
|
||||
|
||||
# First chunk: 1 completion token
|
||||
assert data_chunks[0]["usage"]["completion_tokens"] == 1
|
||||
assert data_chunks[0]["usage"]["total_tokens"] == 4
|
||||
|
||||
# Second chunk: 2 completion tokens (cumulative)
|
||||
assert data_chunks[1]["usage"]["completion_tokens"] == 2
|
||||
assert data_chunks[1]["usage"]["total_tokens"] == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_with_logprobs():
|
||||
"""Streaming with logprobs includes logprob data in each chunk."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
logprobs=[{10: Logprob(logprob=-0.5)}],
|
||||
)
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[20],
|
||||
logprobs=[{20: Logprob(logprob=-1.0)}],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10, logprobs=1),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
parsed = _parse_sse_chunks(chunks)
|
||||
data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
|
||||
|
||||
for dc in data_chunks:
|
||||
lp = dc["choices"][0]["logprobs"]
|
||||
assert lp is not None
|
||||
assert len(lp["content"]) == 1
|
||||
assert lp["content"][0]["token"].startswith("token_id:")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_prompt_tokens_details():
|
||||
"""enable_prompt_tokens_details includes cached_tokens in final usage."""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
num_cached_tokens=2,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine, enable_prompt_tokens_details=True)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
stream_options=StreamOptions(include_usage=True),
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
parsed = _parse_sse_chunks(chunks)
|
||||
# Usage-only chunk (before [DONE])
|
||||
usage_chunk = parsed[-2]
|
||||
assert usage_chunk["choices"] == []
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_prompt_tokens_details_zero_cached():
|
||||
"""enable_prompt_tokens_details includes cached_tokens=0 in final usage.
|
||||
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/44377:
|
||||
zero cached tokens must not be treated as falsy and omitted.
|
||||
"""
|
||||
engine = _mock_engine()
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
yield _make_request_output(
|
||||
"req-1",
|
||||
token_ids=[10],
|
||||
finish_reason="stop",
|
||||
finished=True,
|
||||
num_cached_tokens=0,
|
||||
)
|
||||
|
||||
engine.generate = MagicMock(side_effect=mock_generate)
|
||||
serving = _build_serving_tokens(engine, enable_prompt_tokens_details=True)
|
||||
|
||||
request = GenerateRequest(
|
||||
token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=10),
|
||||
model=MODEL_NAME,
|
||||
stream=True,
|
||||
stream_options=StreamOptions(include_usage=True),
|
||||
)
|
||||
|
||||
response = await serving.serve_tokens(request)
|
||||
chunks = []
|
||||
async for chunk in response:
|
||||
chunks.append(chunk)
|
||||
|
||||
parsed = _parse_sse_chunks(chunks)
|
||||
# Usage-only chunk (before [DONE])
|
||||
usage_chunk = parsed[-2]
|
||||
assert usage_chunk["choices"] == []
|
||||
# Zero cached tokens must be present, not omitted
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"] is not None
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the disagg request/response protocol.
|
||||
|
||||
These tests intentionally avoid spinning up a server — they exercise the
|
||||
pydantic validators on ``GenerateRequest`` directly so they run fast and
|
||||
fail loudly if the validator semantics ever drift.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
def _base_payload() -> dict:
|
||||
return {"token_ids": [1, 2, 3], "sampling_params": {}}
|
||||
|
||||
|
||||
def test_omitted_max_tokens_is_not_provided():
|
||||
"""Body without ``max_tokens`` must surface as 'not provided' so the
|
||||
server can apply its own default instead of the dataclass 16."""
|
||||
req = GenerateRequest.model_validate(_base_payload())
|
||||
# SamplingParams' dataclass default leaks through the parsed instance —
|
||||
# this is exactly the bug the server-side defaulting works around.
|
||||
assert req.sampling_params.max_tokens == 16
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
|
||||
def test_explicit_max_tokens_is_provided():
|
||||
"""Even when the client picks the same value as the dataclass default,
|
||||
it must register as explicitly set so the server won't override it."""
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"max_tokens": 16}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert req.sampling_params.max_tokens == 16
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
payload["sampling_params"] = {"max_tokens": 256}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert req.sampling_params.max_tokens == 256
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
|
||||
def test_other_fields_tracked_independently():
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"temperature": 0.0}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
|
||||
|
||||
def test_json_roundtrip_preserves_provided_keys():
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"temperature": 0.5}
|
||||
req = GenerateRequest.model_validate_json(json.dumps(payload))
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
|
||||
|
||||
def test_internal_instance_construction_treats_all_as_provided():
|
||||
"""When internal callers build ``GenerateRequest`` from a pre-resolved
|
||||
``SamplingParams`` instance, every field is considered explicitly set
|
||||
so server-side defaulting can't clobber values resolved upstream."""
|
||||
sp = SamplingParams(max_tokens=500, temperature=0.0)
|
||||
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
# And keys we never touched should also count as provided in this path.
|
||||
assert req.is_sampling_param_provided("top_p")
|
||||
@@ -1,81 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
|
||||
import httpx
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "TitanML/tiny-mixtral"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
|
||||
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
|
||||
# The published config has sliding_window=4096, which produces
|
||||
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
|
||||
# FullAttentionSpec group, so we override sliding_window=null below.
|
||||
NUM_LOCAL_EXPERTS = 8
|
||||
NUM_EXPERTS_PER_TOK = 2
|
||||
NUM_HIDDEN_LAYERS = 2
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--max-model-len",
|
||||
"256",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--enforce-eager",
|
||||
"--enable-return-routed-experts",
|
||||
"--hf-overrides",
|
||||
'{"sliding_window": null}',
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server: RemoteOpenAIServer):
|
||||
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url=server.url_root,
|
||||
timeout=600,
|
||||
headers=headers,
|
||||
) as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_routed_experts(client):
|
||||
"""Test that /inference/v1/generate returns routed_experts when enabled."""
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
choice = data["choices"][0]
|
||||
|
||||
assert choice["routed_experts"] is not None
|
||||
assert choice["token_ids"] is not None
|
||||
|
||||
# routed_experts is base64-encoded .npy bytes; decode to ndarray.
|
||||
routed_experts = np.load(io.BytesIO(base64.b64decode(choice["routed_experts"])))
|
||||
assert routed_experts.ndim == 3
|
||||
num_tokens, num_layers, topk = routed_experts.shape
|
||||
assert num_tokens > 0
|
||||
assert num_layers == NUM_HIDDEN_LAYERS
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
@@ -1,158 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for multimodal features through the /inference/v1/generate endpoint.
|
||||
|
||||
Mirrors test_serving_tokens.py but exercises the multimodal piping
|
||||
using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render ->
|
||||
/generate -> /detokenize path. Intentionally avoids running the HF
|
||||
processor in the pytest parent process to keep os.fork() in sibling
|
||||
tests (e.g. test_weight_transfer_llm.py) deadlock-free.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from PIL import Image
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
RENDER_ENDPOINT = "/v1/chat/completions/render"
|
||||
DETOKENIZE_ENDPOINT = "/detokenize"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def test_image():
|
||||
return Image.new("RGB", (224, 224), color=(255, 0, 0))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
"--no-enable-prefix-caching",
|
||||
]
|
||||
|
||||
envs = os.environ.copy()
|
||||
envs["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server: RemoteOpenAIServer):
|
||||
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url=server.url_root,
|
||||
timeout=600,
|
||||
headers=headers,
|
||||
) as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_to_generate_roundtrip(client, test_image):
|
||||
"""End-to-end: render a multimodal chat -> feed into generate -> decode.
|
||||
|
||||
All preprocessing and detokenization happens in the server subprocess;
|
||||
the pytest parent never imports transformers or touches torch tensors.
|
||||
"""
|
||||
data_url = encode_image_url(test_image, format="PNG")
|
||||
|
||||
render_payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What color is this image? Answer in one word.",
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
render_resp = await client.post(RENDER_ENDPOINT, json=render_payload)
|
||||
render_resp.raise_for_status()
|
||||
render_data = render_resp.json()
|
||||
|
||||
# Validate render output structure: keys exist and values are non-empty
|
||||
# and well-typed.
|
||||
assert "token_ids" in render_data
|
||||
assert isinstance(render_data["token_ids"], list)
|
||||
assert len(render_data["token_ids"]) > 0
|
||||
assert all(isinstance(t, int) for t in render_data["token_ids"])
|
||||
|
||||
assert "features" in render_data
|
||||
features = render_data["features"]
|
||||
assert features is not None
|
||||
assert isinstance(features, dict)
|
||||
|
||||
assert "mm_hashes" in features
|
||||
assert "image" in features["mm_hashes"]
|
||||
image_hashes = features["mm_hashes"]["image"]
|
||||
assert isinstance(image_hashes, list)
|
||||
assert len(image_hashes) > 0
|
||||
assert all(isinstance(h, str) and h for h in image_hashes)
|
||||
|
||||
assert "mm_placeholders" in features
|
||||
assert "image" in features["mm_placeholders"]
|
||||
image_placeholders = features["mm_placeholders"]["image"]
|
||||
assert isinstance(image_placeholders, list)
|
||||
assert len(image_placeholders) > 0
|
||||
for p in image_placeholders:
|
||||
assert isinstance(p.get("offset"), int)
|
||||
assert isinstance(p.get("length"), int)
|
||||
assert p["length"] > 0
|
||||
|
||||
assert "kwargs_data" in features
|
||||
assert "image" in features["kwargs_data"]
|
||||
assert len(features["kwargs_data"]["image"]) > 0
|
||||
|
||||
# Build generate request from render output
|
||||
generate_payload = render_data
|
||||
generate_payload["sampling_params"] = {
|
||||
"max_tokens": 10,
|
||||
"temperature": 0.0,
|
||||
}
|
||||
|
||||
gen_resp = await client.post(GEN_ENDPOINT, json=generate_payload)
|
||||
gen_resp.raise_for_status()
|
||||
gen_data = gen_resp.json()
|
||||
|
||||
assert "choices" in gen_data
|
||||
assert isinstance(gen_data["choices"], list)
|
||||
assert len(gen_data["choices"]) >= 1
|
||||
choice = gen_data["choices"][0]
|
||||
assert "token_ids" in choice
|
||||
assert isinstance(choice["token_ids"], list)
|
||||
assert len(choice["token_ids"]) > 0
|
||||
assert all(isinstance(t, int) for t in choice["token_ids"])
|
||||
|
||||
detok_resp = await client.post(
|
||||
DETOKENIZE_ENDPOINT,
|
||||
json={"model": MODEL_NAME, "tokens": choice["token_ids"]},
|
||||
)
|
||||
detok_resp.raise_for_status()
|
||||
detok_data = detok_resp.json()
|
||||
assert "prompt" in detok_data
|
||||
text = detok_data["prompt"]
|
||||
assert isinstance(text, str)
|
||||
assert len(text) > 0
|
||||
assert "red" in text.lower(), (
|
||||
f"Expected model to identify the red image, got: {text!r}"
|
||||
)
|
||||
@@ -1,425 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.v1.engine.detokenizer import check_stop_strings
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
GEN_ENDPOINT = "/inference/v1/generate"
|
||||
|
||||
|
||||
def get_vocab_size(model_name):
|
||||
config = ModelConfig(
|
||||
model=model_name,
|
||||
seed=0,
|
||||
dtype="bfloat16",
|
||||
)
|
||||
return config.get_vocab_size()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
return AutoTokenizer.from_pretrained(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def messages():
|
||||
return [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "How many countries are in the EU?"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(request):
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--enforce-eager",
|
||||
# On ROCm (e.g. MI355X/gfx950), bf16 GEMM results can differ by
|
||||
# 1 ULP when the batch dimension (M) changes, because different M
|
||||
# values cause the Tensile backend to select different tile
|
||||
# configurations with different fp32 accumulation orders. With
|
||||
# prefix caching, cache-miss prefills compute all tokens in one
|
||||
# pass (large M) while cache-hit requests compute only the
|
||||
# uncached suffix (small M), seeding a divergence that amplifies
|
||||
# through the residual stream and flips argmax tokens.
|
||||
# See: https://github.com/vllm-project/vllm/issues/33123
|
||||
#
|
||||
# Either disable prefix caching entirely, or enable it with
|
||||
# --deterministic-prefix-caching which forces cache-miss prefills
|
||||
# to split at block boundaries so the suffix GEMM shape is always
|
||||
# identical regardless of cache state.
|
||||
#
|
||||
# Option A: disable prefix caching
|
||||
"--no-enable-prefix-caching",
|
||||
#
|
||||
# Option B: deterministic prefix caching
|
||||
# "--enable-prefix-caching",
|
||||
# "--deterministic-prefix-caching",
|
||||
]
|
||||
|
||||
extra_args = getattr(request, "param", None)
|
||||
if extra_args is not None:
|
||||
args = args + (
|
||||
list(extra_args)
|
||||
if isinstance(extra_args, (list, tuple))
|
||||
else [str(extra_args)]
|
||||
)
|
||||
|
||||
envs = os.environ.copy()
|
||||
# See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3888060787
|
||||
envs["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server: RemoteOpenAIServer):
|
||||
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
|
||||
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
base_url=server.url_root,
|
||||
timeout=600,
|
||||
headers=headers,
|
||||
) as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_endpoint(client):
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert "choices" in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_defaults_max_tokens_when_omitted(client):
|
||||
"""Regression: omitting ``max_tokens`` must not silently cap at 16.
|
||||
|
||||
``SamplingParams.max_tokens`` defaults to 16. Before the server-side
|
||||
defaulting was wired up, every request that didn't set ``max_tokens``
|
||||
truncated mid-generation. The server should now fill it in from
|
||||
``max_model_len - prompt_len`` (matching ``/v1/chat/completions``).
|
||||
"""
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
completion_tokens = len(data["choices"][0]["token_ids"])
|
||||
# max_model_len=1024 in the test fixture, prompt is 3 tokens, so we
|
||||
# should get ~1021 tokens of output (capped at max_model_len boundary).
|
||||
assert completion_tokens > 16, (
|
||||
f"expected server-side default to exceed the legacy 16-token cap, "
|
||||
f"got {completion_tokens}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_stream(client):
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5},
|
||||
"stream": True,
|
||||
}
|
||||
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
|
||||
resp.raise_for_status()
|
||||
chunks = []
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
payload_str = line[len("data: ") :]
|
||||
if payload_str == "[DONE]":
|
||||
break
|
||||
chunks.append(json.loads(payload_str))
|
||||
|
||||
assert len(chunks) > 0
|
||||
# Every chunk has choices with token_ids
|
||||
all_token_ids = []
|
||||
for chunk in chunks:
|
||||
assert "choices" in chunk
|
||||
assert len(chunk["choices"]) == 1
|
||||
choice = chunk["choices"][0]
|
||||
assert "token_ids" in choice
|
||||
assert len(choice["token_ids"]) > 0
|
||||
all_token_ids.extend(choice["token_ids"])
|
||||
|
||||
# Last chunk should have a finish_reason
|
||||
assert chunks[-1]["choices"][0]["finish_reason"] is not None
|
||||
|
||||
# Streaming should produce the same tokens as non-streaming
|
||||
non_stream_resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5, "temperature": 0.0},
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
non_stream_data = non_stream_resp.json()
|
||||
# Just verify we got the right number of tokens
|
||||
assert len(all_token_ids) == len(non_stream_data["choices"][0]["token_ids"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("logprobs_value", [0, 1, 5])
|
||||
async def test_generate_logprobs(client, logprobs_value):
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {
|
||||
"max_tokens": 5,
|
||||
"temperature": 0.0,
|
||||
"logprobs": logprobs_value,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
choice = data["choices"][0]
|
||||
assert choice["logprobs"] is not None
|
||||
logprobs_content = choice["logprobs"]["content"]
|
||||
assert len(logprobs_content) == len(choice["token_ids"])
|
||||
for entry in logprobs_content:
|
||||
assert "logprob" in entry
|
||||
assert len(entry["top_logprobs"]) >= 1
|
||||
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_response_as_chat_completions(client, tokenizer, messages):
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
|
||||
for ignore_eos in [True, False]:
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
# NOTE coordinator will set this to skip detokenization
|
||||
"detokenize": False,
|
||||
"ignore_eos": ignore_eos,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
generate_data = generate_resp.json()
|
||||
gen_token_ids = generate_data["choices"][0]["token_ids"]
|
||||
generate_res = tokenizer.decode(gen_token_ids, skip_special_tokens=True)
|
||||
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"ignore_eos": ignore_eos,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
completions_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
completions_data = completions_resp.json()
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
|
||||
if ignore_eos:
|
||||
# When ignoring EOS, only compare up to the first EOS token
|
||||
# Post-EOS generation is undefined and may differ
|
||||
eos_tokens = {
|
||||
tokenizer.eos_token_id,
|
||||
*getattr_iter(
|
||||
tokenizer,
|
||||
[
|
||||
"extra_special_tokens_ids", # Transformers v5
|
||||
"additional_special_tokens_ids", # Transformers v4
|
||||
],
|
||||
[],
|
||||
),
|
||||
}
|
||||
# Find first EOS in generated tokens
|
||||
eos_pos = None
|
||||
for i, tid in enumerate(gen_token_ids):
|
||||
if tid in eos_tokens:
|
||||
eos_pos = i
|
||||
break
|
||||
if eos_pos is not None:
|
||||
gen_token_ids_truncated = gen_token_ids[:eos_pos]
|
||||
generate_res = tokenizer.decode(
|
||||
gen_token_ids_truncated, skip_special_tokens=True
|
||||
)
|
||||
# Truncate completions_res to same length for comparison
|
||||
completions_res = completions_res[: len(generate_res)]
|
||||
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_string_workflow(client, tokenizer, messages):
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"detokenize": False,
|
||||
# stop strings are only supported when detokenize is True.
|
||||
"stop": ["27 member"],
|
||||
},
|
||||
# TODO stream test is much more interesting
|
||||
"stream": False,
|
||||
}
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
generate_resp.raise_for_status()
|
||||
|
||||
payload["sampling_params"]["stop"] = None
|
||||
generate_resp = await client.post(
|
||||
GEN_ENDPOINT, json=payload, headers={"X-Request-Id": "42"}
|
||||
)
|
||||
generate_data = generate_resp.json()
|
||||
generate_res = tokenizer.decode(
|
||||
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
|
||||
)
|
||||
|
||||
# NOTE This is under the responsibility of the coordinator
|
||||
# stop_checker = StopChecker(
|
||||
# max_model_len=1024, get_tokenizer_for_seq=lambda _: tokenizer
|
||||
# )
|
||||
stop_str, truncate_to = check_stop_strings(
|
||||
generate_res, len(generate_res), ["27 member"], False
|
||||
)
|
||||
assert stop_str == "27 member"
|
||||
# abort request that hit stop string (requires tokens-only mode)
|
||||
# res = await client.post("/abort_requests", json={"request_ids": ["generate-tokens-42"]}) # noqa: E501
|
||||
# res.raise_for_status()
|
||||
generate_res = generate_res[:truncate_to]
|
||||
|
||||
# Get stop_str response from chat completions
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"stop": ["27 member"],
|
||||
"chat_template_kwargs": dict(enable_thinking=False),
|
||||
}
|
||||
completions_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
completions_data = completions_resp.json()
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
assert generate_res == completions_res
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"server",
|
||||
[
|
||||
[
|
||||
"--enable-lora",
|
||||
"--lora-modules",
|
||||
"Alice=charent/self_cognition_Alice",
|
||||
"Bob=charent/self_cognition_Bob",
|
||||
"--max-lora-rank",
|
||||
"64",
|
||||
"--max-cpu-loras",
|
||||
"2",
|
||||
]
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_generate_with_lora_adapter(client, tokenizer, messages):
|
||||
# Verify adapters are listed
|
||||
models_resp = await client.get("/v1/models")
|
||||
models_resp.raise_for_status()
|
||||
models = {m["id"] for m in models_resp.json().get("data", [])}
|
||||
assert {"Alice", "Bob"}.issubset(models)
|
||||
|
||||
# Generate using a LoRA adapter by specifying its name as the model
|
||||
payload = {
|
||||
"model": "Alice",
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {"max_tokens": 5},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
assert "choices" in data
|
||||
|
||||
token_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False, # default with Qwen3
|
||||
return_dict=True, # default with Transformers v5
|
||||
).input_ids
|
||||
payload = {
|
||||
"model": "Alice",
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"detokenize": False,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
generate_data = generate_resp.json()
|
||||
generate_res = tokenizer.decode(
|
||||
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
|
||||
)
|
||||
|
||||
payload = {
|
||||
"model": "Alice",
|
||||
"messages": messages,
|
||||
"max_tokens": 24,
|
||||
"temperature": 0.0,
|
||||
"stream": False,
|
||||
"chat_template_kwargs": dict(enable_thinking=False),
|
||||
}
|
||||
completions_resp = await client.post("/v1/chat/completions", json=payload)
|
||||
completions_data = completions_resp.json()
|
||||
completions_res = completions_data["choices"][0]["message"]["content"]
|
||||
|
||||
assert generate_res == completions_res
|
||||
@@ -1,28 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
||||
from vllm.logprobs import Logprob
|
||||
|
||||
|
||||
def test_top_logprobs_alternatives_have_own_token_ids():
|
||||
"""Each top_logprobs alternative must carry its own token_id placeholder."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[262],
|
||||
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2), 428: Logprob(-2.3)}],
|
||||
num_output_top_logprobs=2,
|
||||
)
|
||||
tokens = {e.token for e in result.content[0].top_logprobs}
|
||||
assert tokens == {"token_id:262", "token_id:257"}, f"got {tokens}"
|
||||
|
||||
|
||||
def test_logprobs_zero_emits_sampled_token():
|
||||
"""logprobs=0 must still emit 1 entry (the sampled token)."""
|
||||
result = ServingTokens._create_tokens_logprobs(
|
||||
None,
|
||||
token_ids=[7],
|
||||
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
|
||||
num_output_top_logprobs=0,
|
||||
)
|
||||
assert len(result.content[0].top_logprobs) == 1
|
||||
@@ -1,924 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for the /derender endpoints (postprocessing counterpart to /render)."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
with RemoteLaunchRenderServer(MODEL_NAME, []) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_for(""), timeout=30.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _render_chat(client: httpx.AsyncClient) -> dict:
|
||||
"""Render a minimal chat request and return the GenerateRequest dict."""
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _make_generate_response(
|
||||
token_ids: list[int] | None,
|
||||
request_id: str = "chatcmpl-test-id",
|
||||
finish_reason: str = "stop",
|
||||
logprobs: dict | None = None,
|
||||
prompt_logprobs: list | None = None,
|
||||
kv_transfer_params: dict | None = None,
|
||||
) -> dict:
|
||||
choice: dict = {
|
||||
"index": 0,
|
||||
"token_ids": token_ids,
|
||||
"finish_reason": finish_reason,
|
||||
"logprobs": logprobs,
|
||||
}
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"choices": [choice],
|
||||
"prompt_logprobs": prompt_logprobs,
|
||||
"kv_transfer_params": kv_transfer_params,
|
||||
}
|
||||
|
||||
|
||||
def _make_logprobs_with_placeholders(token_id: int = 1234) -> dict:
|
||||
entry = {
|
||||
"token": f"token_id:{token_id}",
|
||||
"logprob": -1.0,
|
||||
"bytes": None,
|
||||
"top_logprobs": [
|
||||
{"token": f"token_id:{token_id + 1}", "logprob": -2.0, "bytes": None}
|
||||
],
|
||||
}
|
||||
return {"content": [entry]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Chat derender tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_roundtrip(client):
|
||||
"""Render then derender: decoded content should be a non-empty string."""
|
||||
gen_req = await _render_chat(client)
|
||||
# Use the first 5 rendered token IDs as synthetic "generated" tokens.
|
||||
synthetic_ids = gen_req["token_ids"][:5]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(synthetic_ids),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["object"] == "chat.completion"
|
||||
assert len(data["choices"]) == 1
|
||||
assert data["choices"][0]["message"]["content"]
|
||||
assert data["choices"][0]["message"]["role"] == "assistant"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_usage(client):
|
||||
"""Supplied prompt_tokens flows through into usage correctly."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(synthetic_ids),
|
||||
"prompt_tokens": 10,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
usage = response.json()["usage"]
|
||||
assert usage["prompt_tokens"] == 10
|
||||
assert usage["completion_tokens"] == len(synthetic_ids)
|
||||
assert usage["total_tokens"] == 10 + len(synthetic_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_usage_default(client):
|
||||
"""Omitting prompt_tokens gives usage.prompt_tokens == 0."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(synthetic_ids),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
usage = response.json()["usage"]
|
||||
assert usage["prompt_tokens"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_logprobs(client):
|
||||
"""token_id:N placeholders in content.token are resolved to real strings."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
token_id = synthetic_ids[0]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids,
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
logprobs = data["choices"][0]["logprobs"]
|
||||
assert logprobs is not None
|
||||
content = logprobs["content"]
|
||||
assert content is not None and len(content) == 1
|
||||
token_str = content[0]["token"]
|
||||
assert not token_str.startswith("token_id:"), (
|
||||
f"Placeholder was not resolved: {token_str!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_logprobs_bytes(client):
|
||||
"""Resolved logprob entries have bytes populated as list[int]."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
token_id = synthetic_ids[0]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids,
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
content = response.json()["choices"][0]["logprobs"]["content"]
|
||||
bytes_field = content[0]["bytes"]
|
||||
assert isinstance(bytes_field, list)
|
||||
assert len(bytes_field) > 0
|
||||
assert all(isinstance(b, int) for b in bytes_field)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_top_logprobs(client):
|
||||
"""top_logprobs entries also have their placeholders resolved."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
token_id = synthetic_ids[0]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids,
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
content = response.json()["choices"][0]["logprobs"]["content"]
|
||||
top = content[0]["top_logprobs"]
|
||||
assert len(top) == 1
|
||||
assert not top[0]["token"].startswith("token_id:"), (
|
||||
f"top_logprobs placeholder not resolved: {top[0]['token']!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_prompt_logprobs_passthrough(client):
|
||||
"""prompt_logprobs on GenerateResponse passes through unchanged."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
# prompt_logprobs is a list[dict[int, Logprob] | None]; use None entries.
|
||||
prompt_logprobs = [None, None]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids, prompt_logprobs=prompt_logprobs
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["prompt_logprobs"] == prompt_logprobs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_kv_transfer_params_passthrough(client):
|
||||
"""kv_transfer_params passes through to the ChatCompletionResponse."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
kv = {"key": "value"}
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(
|
||||
synthetic_ids, kv_transfer_params=kv
|
||||
),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["kv_transfer_params"] == kv
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_empty_token_ids(client):
|
||||
"""Empty token_ids list returns 400."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response([]),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_null_token_ids(client):
|
||||
"""Null token_ids returns 400."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_response": _make_generate_response(None),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_chat_unknown_model(client):
|
||||
"""Unknown model returns 404."""
|
||||
gen_req = await _render_chat(client)
|
||||
synthetic_ids = gen_req["token_ids"][:3]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": "does-not-exist",
|
||||
"generate_response": _make_generate_response(synthetic_ids),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Completion derender tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _render_completion(client: httpx.AsyncClient, prompt: str) -> dict:
|
||||
"""Render a completion prompt and return the first GenerateRequest dict."""
|
||||
resp = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={"model": MODEL_NAME, "prompt": prompt},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert isinstance(data, list) and len(data) >= 1
|
||||
return data[0]
|
||||
|
||||
|
||||
def _make_completion_generate_response(
|
||||
token_ids: list[int],
|
||||
request_id: str,
|
||||
kv_transfer_params: dict | None = None,
|
||||
logprobs: dict | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": token_ids,
|
||||
"finish_reason": "stop",
|
||||
"logprobs": logprobs,
|
||||
}
|
||||
],
|
||||
"prompt_logprobs": None,
|
||||
"kv_transfer_params": kv_transfer_params,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_roundtrip(client):
|
||||
"""Two prompts rendered, two GenerateResponses → two choices with indices 0, 1."""
|
||||
gr1 = await _render_completion(client, "Hello world")
|
||||
gr2 = await _render_completion(client, "Goodbye world")
|
||||
|
||||
ids1 = gr1["token_ids"][:4]
|
||||
ids2 = gr2["token_ids"][:4]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(ids1, gr1["request_id"]),
|
||||
_make_completion_generate_response(ids2, gr2["request_id"]),
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["object"] == "text_completion"
|
||||
choices = data["choices"]
|
||||
assert len(choices) == 2
|
||||
assert choices[0]["index"] == 0
|
||||
assert choices[1]["index"] == 1
|
||||
assert choices[0]["text"]
|
||||
assert choices[1]["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_usage_aggregation(client):
|
||||
"""prompt_tokens=[5, 10] is aggregated correctly into usage."""
|
||||
gr1 = await _render_completion(client, "Hello")
|
||||
gr2 = await _render_completion(client, "World")
|
||||
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
ids2 = gr2["token_ids"][:4]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(ids1, gr1["request_id"]),
|
||||
_make_completion_generate_response(ids2, gr2["request_id"]),
|
||||
],
|
||||
"prompt_tokens": [5, 10],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
usage = response.json()["usage"]
|
||||
assert usage["prompt_tokens"] == 15
|
||||
assert usage["completion_tokens"] == len(ids1) + len(ids2)
|
||||
assert usage["total_tokens"] == 15 + len(ids1) + len(ids2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_prompt_tokens_length_mismatch(client):
|
||||
"""len(prompt_tokens) != len(generate_responses) returns 400."""
|
||||
gr1 = await _render_completion(client, "Hello")
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(ids1, gr1["request_id"]),
|
||||
],
|
||||
"prompt_tokens": [5, 10],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_empty_generate_responses(client):
|
||||
"""Empty generate_responses list returns 400."""
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={"model": MODEL_NAME, "generate_responses": []},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_logprobs(client):
|
||||
"""token_id:N placeholders in logprobs are resolved; CompletionLogProbs
|
||||
flat-list structure is returned with non-empty tokens and text_offsets."""
|
||||
gr1 = await _render_completion(client, "Hello world")
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
token_id = ids1[0]
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(
|
||||
ids1,
|
||||
gr1["request_id"],
|
||||
logprobs=_make_logprobs_with_placeholders(token_id),
|
||||
),
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
logprobs = response.json()["choices"][0]["logprobs"]
|
||||
assert logprobs is not None
|
||||
tokens = logprobs["tokens"]
|
||||
assert len(tokens) == 1
|
||||
assert not tokens[0].startswith("token_id:"), (
|
||||
f"Placeholder was not resolved: {tokens[0]!r}"
|
||||
)
|
||||
assert len(logprobs["token_logprobs"]) == 1
|
||||
assert isinstance(logprobs["token_logprobs"][0], float)
|
||||
assert len(logprobs["text_offset"]) == 1
|
||||
assert logprobs["text_offset"][0] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_derender_completion_kv_transfer_params_passthrough(client):
|
||||
"""kv_transfer_params passes through to CompletionResponse."""
|
||||
gr1 = await _render_completion(client, "Hello")
|
||||
ids1 = gr1["token_ids"][:3]
|
||||
kv = {"node": "abc"}
|
||||
|
||||
response = await client.post(
|
||||
"/v1/completions/derender",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"generate_responses": [
|
||||
_make_completion_generate_response(
|
||||
ids1, gr1["request_id"], kv_transfer_params=kv
|
||||
),
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["kv_transfer_params"] == kv
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# E2E: render -> derender roundtrip with parser (reasoning + tool calls)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PARSER_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
|
||||
|
||||
_E2E_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a city",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser_server():
|
||||
args = [
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
"hermes",
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
]
|
||||
with RemoteLaunchRenderServer(PARSER_MODEL, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def parser_client(parser_server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=parser_server.url_for(""), timeout=60.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser_tokenizer():
|
||||
return get_tokenizer(PARSER_MODEL)
|
||||
|
||||
|
||||
def _encode(tokenizer, text: str) -> list[int]:
|
||||
return tokenizer.encode(text, add_special_tokens=False)
|
||||
|
||||
|
||||
def _decoded(tokenizer, token_ids: list[int]) -> str:
|
||||
return tokenizer.decode(token_ids, skip_special_tokens=True)
|
||||
|
||||
|
||||
def _require_markers_survive(tokenizer, text: str, *markers: str) -> list[int]:
|
||||
"""Encode text and skip the test if any marker is lost in roundtrip."""
|
||||
ids = _encode(tokenizer, text)
|
||||
decoded = tokenizer.decode(ids, skip_special_tokens=False)
|
||||
for m in markers:
|
||||
if m not in decoded:
|
||||
pytest.skip(f"Marker {m!r} lost in encode->decode roundtrip")
|
||||
return ids
|
||||
|
||||
|
||||
async def _e2e_render_chat(
|
||||
client: httpx.AsyncClient,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
) -> dict:
|
||||
resp = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={"model": model, "messages": messages},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _e2e_generate_response(
|
||||
token_ids: list[int],
|
||||
request_id: str = "chatcmpl-e2e-test",
|
||||
) -> dict:
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"token_ids": token_ids,
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_plain_roundtrip(parser_client, parser_tokenizer):
|
||||
"""Plain text without reasoning markers roundtrips correctly."""
|
||||
messages = [{"role": "user", "content": "What is 2+2?"}]
|
||||
gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
|
||||
|
||||
answer = "The answer is four."
|
||||
output_ids = _encode(parser_tokenizer, answer)
|
||||
expected = _decoded(parser_tokenizer, output_ids)
|
||||
|
||||
resp = await parser_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": PARSER_MODEL,
|
||||
"generate_response": _e2e_generate_response(output_ids),
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
content = resp.json()["choices"][0]["message"]["content"]
|
||||
assert content == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_token_identity(parser_client, parser_tokenizer):
|
||||
"""encode(derender(token_ids)) == token_ids (RL invariant)."""
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
|
||||
|
||||
answer = "Hello! How can I help?"
|
||||
output_ids = _encode(parser_tokenizer, answer)
|
||||
|
||||
resp = await parser_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": PARSER_MODEL,
|
||||
"generate_response": _e2e_generate_response(output_ids),
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
content = resp.json()["choices"][0]["message"]["content"]
|
||||
re_encoded = _encode(parser_tokenizer, content)
|
||||
assert output_ids == re_encoded
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_non_ascii_roundtrip(parser_client, parser_tokenizer):
|
||||
"""CJK + emoji roundtrip without U+FFFD."""
|
||||
messages = [{"role": "user", "content": "Reply in Chinese"}]
|
||||
gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
|
||||
|
||||
answer = "你好世界 😀"
|
||||
output_ids = _encode(parser_tokenizer, answer)
|
||||
|
||||
resp = await parser_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": PARSER_MODEL,
|
||||
"generate_response": _e2e_generate_response(output_ids),
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
content = resp.json()["choices"][0]["message"]["content"]
|
||||
assert "�" not in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_parsed_reasoning(parser_client, parser_tokenizer):
|
||||
"""<think>...</think> splits into reasoning + content."""
|
||||
messages = [{"role": "user", "content": "What is 2+3?"}]
|
||||
gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
|
||||
|
||||
reasoning_text = "The user wants 2 plus 3. That is 5."
|
||||
answer_text = "The answer is 5."
|
||||
output_text = f"<think>{reasoning_text}</think>{answer_text}"
|
||||
output_ids = _require_markers_survive(parser_tokenizer, output_text, "</think>")
|
||||
|
||||
resp = await parser_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": PARSER_MODEL,
|
||||
"generate_response": _e2e_generate_response(output_ids),
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
"chat_request": {
|
||||
"model": PARSER_MODEL,
|
||||
"messages": messages,
|
||||
"include_reasoning": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
msg = resp.json()["choices"][0]["message"]
|
||||
assert msg["reasoning"] is not None
|
||||
assert reasoning_text in msg["reasoning"]
|
||||
assert answer_text in msg["content"]
|
||||
assert "<think>" not in msg["content"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_parsed_tool_call(parser_client, parser_tokenizer):
|
||||
"""<tool_call> extracted into tool_calls field."""
|
||||
messages = [{"role": "user", "content": "Weather in Paris?"}]
|
||||
gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
|
||||
|
||||
output_text = (
|
||||
"<think>Let me check the weather.</think>"
|
||||
'<tool_call>\n{"name": "get_weather", '
|
||||
'"arguments": {"city": "Paris"}}\n</tool_call>'
|
||||
)
|
||||
output_ids = _require_markers_survive(
|
||||
parser_tokenizer,
|
||||
output_text,
|
||||
"</think>",
|
||||
"<tool_call>",
|
||||
"</tool_call>",
|
||||
)
|
||||
|
||||
resp = await parser_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": PARSER_MODEL,
|
||||
"generate_response": _e2e_generate_response(output_ids),
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
"chat_request": {
|
||||
"model": PARSER_MODEL,
|
||||
"messages": messages,
|
||||
"tools": _E2E_TOOLS,
|
||||
"tool_choice": "auto",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
choice = resp.json()["choices"][0]
|
||||
assert choice["message"]["tool_calls"]
|
||||
assert choice["message"]["tool_calls"][0]["function"]["name"] == "get_weather"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_parsed_reasoning_and_tool_call(parser_client, parser_tokenizer):
|
||||
"""Reasoning + tool call in the same output."""
|
||||
messages = [{"role": "user", "content": "Weather in Paris?"}]
|
||||
gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
|
||||
|
||||
reasoning_text = "I should look up the weather."
|
||||
tool_text = (
|
||||
'<tool_call>\n{"name": "get_weather", '
|
||||
'"arguments": {"city": "Paris"}}\n</tool_call>'
|
||||
)
|
||||
output_text = f"<think>{reasoning_text}</think>{tool_text}"
|
||||
output_ids = _require_markers_survive(
|
||||
parser_tokenizer, output_text, "</think>", "<tool_call>"
|
||||
)
|
||||
|
||||
resp = await parser_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": PARSER_MODEL,
|
||||
"generate_response": _e2e_generate_response(output_ids),
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
"chat_request": {
|
||||
"model": PARSER_MODEL,
|
||||
"messages": messages,
|
||||
"tools": _E2E_TOOLS,
|
||||
"tool_choice": "auto",
|
||||
"include_reasoning": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
choice = resp.json()["choices"][0]
|
||||
assert choice["message"]["reasoning"] is not None
|
||||
assert reasoning_text in choice["message"]["reasoning"]
|
||||
assert choice["message"]["tool_calls"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_no_chat_request_fallback(parser_client, parser_tokenizer):
|
||||
"""Without chat_request, derender falls back to plain detokenization."""
|
||||
messages = [{"role": "user", "content": "Hello"}]
|
||||
gen_req = await _e2e_render_chat(parser_client, PARSER_MODEL, messages)
|
||||
|
||||
answer = "Hi there!"
|
||||
output_ids = _encode(parser_tokenizer, answer)
|
||||
|
||||
resp = await parser_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": PARSER_MODEL,
|
||||
"generate_response": _e2e_generate_response(output_ids),
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
content = resp.json()["choices"][0]["message"]["content"]
|
||||
assert "Hi" in content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# E2E: HarmonyParser + GPT-OSS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
HARMONY_MODEL = "openai/gpt-oss-20b"
|
||||
|
||||
|
||||
def _ensure_harmony_vocab():
|
||||
"""Pre-cache the o200k_base BPE file needed by openai-harmony.
|
||||
|
||||
The Rust tiktoken-rs backend downloads from Azure Blob Storage, which
|
||||
may be unreachable in some environments. When the cache is cold we
|
||||
fetch the file ourselves and place it in ``/tmp/tiktoken-rs-cache/``
|
||||
using the SHA-1(URL) filename that tiktoken-rs expects.
|
||||
"""
|
||||
import hashlib
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
url = "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken"
|
||||
cache_dir = Path("/tmp/tiktoken-rs-cache")
|
||||
cache_key = hashlib.sha1(url.encode()).hexdigest()
|
||||
cache_file = cache_dir / cache_key
|
||||
if not cache_file.exists():
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
urllib.request.urlretrieve(url, cache_file)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def harmony_server():
|
||||
_ensure_harmony_vocab()
|
||||
args = [
|
||||
"--trust-remote-code",
|
||||
"--enable-auto-tool-choice",
|
||||
"--tool-call-parser",
|
||||
"openai",
|
||||
"--reasoning-parser",
|
||||
"openai_gptoss",
|
||||
]
|
||||
with RemoteLaunchRenderServer(HARMONY_MODEL, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def harmony_client(harmony_server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=harmony_server.url_for(""), timeout=60.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def harmony_tokenizer():
|
||||
return get_tokenizer(HARMONY_MODEL, trust_remote_code=True)
|
||||
|
||||
|
||||
def _harmony_extract_assistant_ids(
|
||||
tokenizer, assistant_msg: dict, user_content: str = "test"
|
||||
) -> list[int]:
|
||||
"""Extract assistant token IDs via apply_chat_template diff."""
|
||||
prompt = [{"role": "user", "content": user_content}]
|
||||
full = prompt + [assistant_msg]
|
||||
text_prompt = tokenizer.apply_chat_template(
|
||||
prompt, add_generation_prompt=True, tokenize=False
|
||||
)
|
||||
text_full = tokenizer.apply_chat_template(
|
||||
full, add_generation_prompt=False, tokenize=False
|
||||
)
|
||||
prompt_ids = tokenizer.encode(text_prompt)
|
||||
full_ids = tokenizer.encode(text_full)
|
||||
assistant_ids = list(full_ids[len(prompt_ids) :])
|
||||
if not assistant_ids:
|
||||
pytest.skip("Could not extract assistant tokens for Harmony")
|
||||
return assistant_ids
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_harmony_plain_roundtrip(harmony_client, harmony_tokenizer):
|
||||
"""GPT-OSS content-only roundtrip."""
|
||||
messages = [{"role": "user", "content": "What is 2+2?"}]
|
||||
gen_req = await _e2e_render_chat(harmony_client, HARMONY_MODEL, messages)
|
||||
|
||||
assistant_msg = {"role": "assistant", "content": "Four."}
|
||||
output_ids = _harmony_extract_assistant_ids(harmony_tokenizer, assistant_msg)
|
||||
|
||||
resp = await harmony_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": HARMONY_MODEL,
|
||||
"generate_response": _e2e_generate_response(output_ids),
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
"chat_request": {
|
||||
"model": HARMONY_MODEL,
|
||||
"messages": messages,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
content = resp.json()["choices"][0]["message"]["content"]
|
||||
assert content is not None and len(content) > 0
|
||||
assert "Four" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_e2e_harmony_reasoning(harmony_client, harmony_tokenizer):
|
||||
"""GPT-OSS reasoning: analysis channel extracted."""
|
||||
messages = [{"role": "user", "content": "Add 2 and 3."}]
|
||||
gen_req = await _e2e_render_chat(harmony_client, HARMONY_MODEL, messages)
|
||||
|
||||
reasoning_text = "The user wants 2 plus 3."
|
||||
answer_text = "The answer is 5."
|
||||
assistant_msg = {
|
||||
"role": "assistant",
|
||||
"thinking": reasoning_text,
|
||||
"content": answer_text,
|
||||
}
|
||||
output_ids = _harmony_extract_assistant_ids(harmony_tokenizer, assistant_msg)
|
||||
|
||||
decoded = harmony_tokenizer.decode(output_ids)
|
||||
if reasoning_text not in decoded:
|
||||
pytest.skip("Harmony template did not render thinking")
|
||||
|
||||
resp = await harmony_client.post(
|
||||
"/v1/chat/completions/derender",
|
||||
json={
|
||||
"model": HARMONY_MODEL,
|
||||
"generate_response": _e2e_generate_response(output_ids),
|
||||
"prompt_tokens": len(gen_req["token_ids"]),
|
||||
"chat_request": {
|
||||
"model": HARMONY_MODEL,
|
||||
"messages": messages,
|
||||
"include_reasoning": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
msg = resp.json()["choices"][0]["message"]
|
||||
assert msg["reasoning"] is not None
|
||||
assert reasoning_text in msg["reasoning"]
|
||||
assert answer_text in (msg["content"] or "")
|
||||
@@ -1,179 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E tests for render endpoints via `vllm launch` (GPU-less serving)."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args: list[str] = []
|
||||
with RemoteLaunchRenderServer(MODEL_NAME, args, max_wait_seconds=120) as srv:
|
||||
yield srv
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_for(""), timeout=30.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
# -- Chat Completion Render --
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_basic(client):
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hello, how are you?"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Response should be a GenerateRequest dict
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
assert all(isinstance(t, int) for t in data["token_ids"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_multi_turn(client):
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
|
||||
|
||||
# -- Completion Render --
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_basic(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "Once upon a time",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, list)
|
||||
assert len(data) > 0
|
||||
|
||||
first_prompt = data[0]
|
||||
assert "token_ids" in first_prompt
|
||||
assert "sampling_params" in first_prompt
|
||||
assert "model" in first_prompt
|
||||
assert "request_id" in first_prompt
|
||||
assert isinstance(first_prompt["token_ids"], list)
|
||||
assert len(first_prompt["token_ids"]) > 0
|
||||
assert first_prompt["request_id"].startswith("cmpl-")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_multiple_prompts(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": ["Hello world", "Goodbye world"],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert isinstance(data, list)
|
||||
assert len(data) == 2
|
||||
|
||||
for prompt in data:
|
||||
assert "token_ids" in prompt
|
||||
assert "sampling_params" in prompt
|
||||
assert "model" in prompt
|
||||
assert "request_id" in prompt
|
||||
assert len(prompt["token_ids"]) > 0
|
||||
assert prompt["request_id"].startswith("cmpl-")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_invalid_model(client):
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": "nonexistent-model",
|
||||
"prompt": "Hello",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert "error" in response.json()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_is_fast(client):
|
||||
"""Render should complete quickly since there is no inference."""
|
||||
import time
|
||||
|
||||
start = time.perf_counter()
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "Tell me a very long story about " * 10,
|
||||
},
|
||||
)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
assert response.status_code == 200
|
||||
assert elapsed < 2.0
|
||||
|
||||
|
||||
# -- Health & Models --
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint(client):
|
||||
response = await client.get("/health")
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_models_endpoint(client):
|
||||
response = await client.get("/v1/models")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "data" in data
|
||||
model_ids = [m["id"] for m in data["data"]]
|
||||
assert MODEL_NAME in model_ids
|
||||
@@ -1,495 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for the /render endpoints that expose prompt preprocessing."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args: list[str] = ["--trust-request-chat-template"]
|
||||
|
||||
with RemoteLaunchRenderServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_for(""), timeout=30.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_basic(client):
|
||||
"""Test basic completion render endpoint."""
|
||||
# Make request to render endpoint
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "When should a chat-completions handler return an empty string?",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify response structure - list of GenerateRequest
|
||||
assert isinstance(data, list)
|
||||
assert len(data) > 0
|
||||
|
||||
# Verify first prompt is a GenerateRequest
|
||||
first_prompt = data[0]
|
||||
assert "token_ids" in first_prompt
|
||||
assert "sampling_params" in first_prompt
|
||||
assert "model" in first_prompt
|
||||
assert "request_id" in first_prompt
|
||||
assert isinstance(first_prompt["token_ids"], list)
|
||||
assert len(first_prompt["token_ids"]) > 0
|
||||
assert first_prompt["model"] == MODEL_NAME
|
||||
assert first_prompt["request_id"].startswith("cmpl-")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_basic(client):
|
||||
"""Test basic chat completion render endpoint."""
|
||||
# Make request to render endpoint
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Returning an empty string for the prompt may be confusing."
|
||||
),
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify response structure - should be a GenerateRequest
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
|
||||
# Verify token IDs are integers and BOS token is present
|
||||
token_ids = data["token_ids"]
|
||||
assert all(isinstance(tid, int) for tid in token_ids)
|
||||
assert token_ids[0] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_multiple_prompts(client):
|
||||
"""Test completion render with multiple prompts."""
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": ["Hello world", "Goodbye world"],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Should return two GenerateRequest items
|
||||
assert isinstance(data, list)
|
||||
assert len(data) == 2
|
||||
|
||||
# Verify both prompts have GenerateRequest fields
|
||||
for prompt in data:
|
||||
assert "token_ids" in prompt
|
||||
assert "sampling_params" in prompt
|
||||
assert "model" in prompt
|
||||
assert "request_id" in prompt
|
||||
assert len(prompt["token_ids"]) > 0
|
||||
assert prompt["request_id"].startswith("cmpl-")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_multi_turn(client):
|
||||
"""Test chat completion render with multi-turn conversation."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify tokenization occurred
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_with_stream_true(client):
|
||||
"""Render accepts stream params but still returns JSON (non-streamed)."""
|
||||
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"stream": True,
|
||||
"stream_options": {
|
||||
"include_usage": True,
|
||||
"continuous_usage_stats": True,
|
||||
},
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Stream options should be accepted by /render.",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers.get("content-type", "").startswith("application/json")
|
||||
|
||||
data = response.json()
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
|
||||
# /render should preserve stream fields on the returned token-in request.
|
||||
assert data.get("stream") is True
|
||||
assert isinstance(data.get("stream_options"), dict)
|
||||
assert data["stream_options"].get("include_usage") is True
|
||||
assert data["stream_options"].get("continuous_usage_stats") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_error_invalid_model(client):
|
||||
"""Test completion render with invalid model returns error."""
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": "invalid-model-name",
|
||||
"prompt": "Hello",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
data = response.json()
|
||||
assert "error" in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_error_invalid_model(client):
|
||||
"""Test chat completion render with invalid model returns error."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": "invalid-model-name",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 404
|
||||
data = response.json()
|
||||
assert "error" in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_no_generation(client):
|
||||
"""Verify render endpoint does not generate text."""
|
||||
# This test verifies that calling render is fast (no generation)
|
||||
import time
|
||||
|
||||
start = time.perf_counter()
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "Tell me a very long story about " * 10,
|
||||
},
|
||||
)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
assert response.status_code == 200
|
||||
# Render should be fast (< 1 second) since no generation
|
||||
assert elapsed < 1.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_with_sampling_params(client):
|
||||
"""Verify sampling params are correctly returned by /render."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Test sampling params"}],
|
||||
"temperature": 0.123,
|
||||
"top_p": 0.456,
|
||||
"frequency_penalty": 1.1,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
assert "sampling_params" in data
|
||||
sampling_params = data["sampling_params"]
|
||||
|
||||
assert sampling_params.get("temperature") == 0.123
|
||||
assert sampling_params.get("top_p") == 0.456
|
||||
assert sampling_params.get("frequency_penalty") == 1.1
|
||||
|
||||
# Check that internal fields are not present
|
||||
assert "_all_stop_token_ids" not in sampling_params
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_emits_token_offsets(client):
|
||||
"""With return_token_offsets, /v1/completions/render returns per-token
|
||||
(start, end) char offsets aligned with token_ids."""
|
||||
prompt = "Hello, world."
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": prompt,
|
||||
"return_token_offsets": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert isinstance(data, list)
|
||||
offsets = data[0]["token_offsets"]
|
||||
assert offsets is not None
|
||||
assert len(offsets) == len(data[0]["token_ids"])
|
||||
for start, end in offsets:
|
||||
assert isinstance(start, int) and isinstance(end, int)
|
||||
assert 0 <= start <= end <= len(prompt)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_default_no_token_offsets(client):
|
||||
"""Without the flag, token_offsets must be null (existing responses
|
||||
unchanged)."""
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": "Hello, world.",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data[0]["token_offsets"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_emits_token_offsets(client):
|
||||
"""With return_token_offsets, /v1/chat/completions/render returns
|
||||
per-token offsets relative to the templated prompt string."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hello, world."}],
|
||||
"return_token_offsets": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert isinstance(data, dict)
|
||||
offsets = data["token_offsets"]
|
||||
assert offsets is not None
|
||||
assert len(offsets) == len(data["token_ids"])
|
||||
for start, end in offsets:
|
||||
assert isinstance(start, int) and isinstance(end, int)
|
||||
assert 0 <= start <= end
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_default_no_token_offsets(client):
|
||||
"""Without the flag, chat render token_offsets must be null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hello, world."}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["token_offsets"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_multiple_prompts_token_offsets(client):
|
||||
"""Each prompt in a batch gets its own offsets aligned with its tokens."""
|
||||
prompts = ["Hello, world.", "Goodbye, world."]
|
||||
response = await client.post(
|
||||
"/v1/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"prompt": prompts,
|
||||
"return_token_offsets": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert len(data) == len(prompts)
|
||||
for item, prompt in zip(data, prompts):
|
||||
offsets = item["token_offsets"]
|
||||
assert offsets is not None
|
||||
assert len(offsets) == len(item["token_ids"])
|
||||
for start, end in offsets:
|
||||
assert 0 <= start <= end <= len(prompt)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_assistant_tokens_mask_default(client):
|
||||
"""Without return_assistant_tokens_mask, assistant_tokens_mask should be null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi!"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data.get("assistant_tokens_mask") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_assistant_tokens_mask_false(client):
|
||||
"""Explicitly setting return_assistant_tokens_mask=false gives null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
],
|
||||
"return_assistant_tokens_mask": False,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data.get("assistant_tokens_mask") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_render_assistant_tokens_mask_null_without_gen_tags(
|
||||
client,
|
||||
):
|
||||
"""The tiny test model lacks ``{% generation %}`` tags, so the mask is null."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi!"},
|
||||
],
|
||||
"return_assistant_tokens_mask": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json().get("assistant_tokens_mask") is None
|
||||
|
||||
|
||||
# A minimal chat template with {% generation %} tags so we can test that
|
||||
# the mask correctly marks assistant tokens.
|
||||
_TEMPLATE_WITH_GENERATION = (
|
||||
"{% for m in messages %}"
|
||||
"{% if m['role'] == 'user' %}User: {{ m['content'] }}\n"
|
||||
"{% elif m['role'] == 'assistant' %}"
|
||||
"{% generation %}Assistant: {{ m['content'] }}\n{% endgeneration %}"
|
||||
"{% endif %}"
|
||||
"{% endfor %}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags(
|
||||
client,
|
||||
):
|
||||
"""With a ``{% generation %}``-enabled template, the mask marks assistant
|
||||
tokens and the masked tokens decode to the assistant content."""
|
||||
response = await client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi!"},
|
||||
{"role": "user", "content": "Bye"},
|
||||
],
|
||||
"chat_template": _TEMPLATE_WITH_GENERATION,
|
||||
"return_assistant_tokens_mask": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
mask = data["assistant_tokens_mask"]
|
||||
token_ids = data["token_ids"]
|
||||
assert mask is not None
|
||||
assert isinstance(mask, list)
|
||||
assert len(mask) == len(token_ids)
|
||||
assert all(v in (0, 1) for v in mask)
|
||||
assert sum(mask) > 0, "mask should mark at least one assistant token"
|
||||
|
||||
# Detokenize masked (assistant) and unmasked (non-assistant) tokens
|
||||
# separately to verify the mask is correct, not just non-empty.
|
||||
masked_ids = [t for t, m in zip(token_ids, mask, strict=True) if m]
|
||||
unmasked_ids = [t for t, m in zip(token_ids, mask, strict=True) if not m]
|
||||
|
||||
detok = await client.post(
|
||||
"/detokenize",
|
||||
json={"model": MODEL_NAME, "tokens": masked_ids},
|
||||
)
|
||||
assert detok.status_code == 200
|
||||
assert "Hi!" in detok.json()["prompt"]
|
||||
|
||||
detok_rest = await client.post(
|
||||
"/detokenize",
|
||||
json={"model": MODEL_NAME, "tokens": unmasked_ids},
|
||||
)
|
||||
assert detok_rest.status_code == 200
|
||||
assert "Hi!" not in detok_rest.json()["prompt"]
|
||||
assert "Bye" in detok_rest.json()["prompt"]
|
||||
@@ -1,155 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Multimodal tests for the /render endpoints that expose prompt preprocessing."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
|
||||
VISION_MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vision_server():
|
||||
"""Vision-capable server used for multimodal /render tests."""
|
||||
|
||||
args = [
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"100",
|
||||
"--max-num-seqs",
|
||||
"1",
|
||||
"--limit-mm-per-prompt.image",
|
||||
"1",
|
||||
"--limit-mm-per-prompt.video",
|
||||
"0",
|
||||
]
|
||||
|
||||
env_overrides: dict[str, str] = {}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
VISION_MODEL_NAME,
|
||||
args,
|
||||
env_dict=env_overrides,
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def vision_client(vision_server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=vision_server.url_for(""), timeout=60.0
|
||||
) as http_client:
|
||||
yield http_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completion_render_with_base64_image_url(
|
||||
vision_client,
|
||||
local_asset_server,
|
||||
):
|
||||
"""Render a multimodal chat request and verify tokens are returned."""
|
||||
|
||||
image = local_asset_server.get_image_asset("RGBA_comp.png")
|
||||
data_url = encode_image_url(image, format="PNG")
|
||||
|
||||
assert data_url.startswith("data:image/")
|
||||
assert ";base64," in data_url
|
||||
|
||||
response = await vision_client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": VISION_MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
data = response.json()
|
||||
assert isinstance(data, dict)
|
||||
assert "token_ids" in data
|
||||
assert isinstance(data["token_ids"], list)
|
||||
assert len(data["token_ids"]) > 0
|
||||
|
||||
# Verify multimodal features are populated
|
||||
assert "features" in data
|
||||
features = data["features"]
|
||||
assert features is not None
|
||||
|
||||
# mm_hashes: should have an "image" key with a list of hash strings
|
||||
assert "mm_hashes" in features
|
||||
assert "image" in features["mm_hashes"]
|
||||
image_hashes = features["mm_hashes"]["image"]
|
||||
assert isinstance(image_hashes, list)
|
||||
assert len(image_hashes) > 0
|
||||
assert all(isinstance(h, str) for h in image_hashes)
|
||||
|
||||
# mm_placeholders: should have an "image" key with offset/length dicts
|
||||
assert "mm_placeholders" in features
|
||||
assert "image" in features["mm_placeholders"]
|
||||
image_placeholders = features["mm_placeholders"]["image"]
|
||||
assert isinstance(image_placeholders, list)
|
||||
assert len(image_placeholders) > 0
|
||||
for p in image_placeholders:
|
||||
assert "offset" in p
|
||||
assert "length" in p
|
||||
assert isinstance(p["offset"], int)
|
||||
assert isinstance(p["length"], int)
|
||||
assert p["length"] > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokenize_matches_render_for_multimodal_input(
|
||||
vision_client,
|
||||
local_asset_server,
|
||||
):
|
||||
"""`/tokenize` should match `/v1/chat/completions/render` token output."""
|
||||
|
||||
image = local_asset_server.get_image_asset("RGBA_comp.png")
|
||||
data_url = encode_image_url(image, format="PNG")
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
render_response = await vision_client.post(
|
||||
"/v1/chat/completions/render",
|
||||
json={
|
||||
"model": VISION_MODEL_NAME,
|
||||
"messages": messages,
|
||||
},
|
||||
)
|
||||
assert render_response.status_code == 200
|
||||
render_data = render_response.json()
|
||||
|
||||
tokenize_response = await vision_client.post(
|
||||
"/tokenize",
|
||||
json={
|
||||
"model": VISION_MODEL_NAME,
|
||||
"messages": messages,
|
||||
},
|
||||
)
|
||||
assert tokenize_response.status_code == 200
|
||||
tokenize_data = tokenize_response.json()
|
||||
|
||||
assert tokenize_data["tokens"] == render_data["token_ids"]
|
||||
assert tokenize_data["count"] == len(render_data["token_ids"])
|
||||
Reference in New Issue
Block a user