[Frontend] Consolidate scale out entrypoints (#44512)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-06-29 03:18:53 -07:00
committed by GitHub
parent eddfd4cf21
commit 3483240b7e
45 changed files with 422 additions and 289 deletions
@@ -0,0 +1,554 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass, field
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.engine.protocol import StreamOptions
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateRequest,
GenerateResponse,
)
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.async_llm import AsyncLLM
MODEL_NAME = "openai-community/gpt2"
BASE_MODEL_PATHS = [
BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
]
@dataclass
class MockHFConfig:
model_type: str = "any"
@dataclass
class MockModelConfig:
task = "generate"
runner_type = "generate"
model = MODEL_NAME
tokenizer = MODEL_NAME
trust_remote_code = False
tokenizer_mode = "auto"
max_model_len = 100
tokenizer_revision = None
multimodal_config = MultiModalConfig()
hf_config = MockHFConfig()
hf_text_config = MockHFConfig()
logits_processors: list[str] | None = None
diff_sampling_param: dict | None = None
allowed_local_media_path: str = ""
allowed_media_domains: list[str] | None = None
encoder_config = None
generation_config: str = "auto"
media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
skip_tokenizer_init = False
is_encoder_decoder: bool = False
is_multimodal_model: bool = False
renderer_num_workers: int = 1
def get_diff_sampling_param(self):
return self.diff_sampling_param or {}
@dataclass
class MockParallelConfig:
_api_process_rank: int = 0
@dataclass
class MockSchedulerConfig:
max_num_seqs: int = 128
@dataclass
class MockVllmConfig:
model_config: MockModelConfig
parallel_config: MockParallelConfig
scheduler_config: MockSchedulerConfig = field(default_factory=MockSchedulerConfig)
def _build_renderer(model_config: MockModelConfig):
return renderer_from_config(
MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
)
def _build_serving_tokens(engine: AsyncLLM, **kwargs) -> ServingTokens:
models = OpenAIServingModels(
engine_client=engine,
base_model_paths=BASE_MODEL_PATHS,
)
online_renderer = OnlineRenderer(
model_config=engine.model_config,
renderer=engine.renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
serving = ServingTokens(
engine,
models,
online_renderer=online_renderer,
request_logger=None,
**kwargs,
)
async def _fake_preprocess(*args, **kwargs):
return [{"prompt_token_ids": [1, 2, 3]}]
serving.online_renderer.preprocess_completion = AsyncMock(
side_effect=_fake_preprocess
)
return serving
def _make_request_output(
request_id: str,
token_ids: list[int],
finish_reason: str | None = None,
finished: bool = False,
prompt_token_ids: list[int] | None = None,
logprobs: list[dict[int, Any] | None] | None = None,
num_cached_tokens: int | None = None,
index: int = 0,
) -> RequestOutput:
return RequestOutput(
request_id=request_id,
prompt=None,
prompt_token_ids=prompt_token_ids or [1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=index,
text="",
token_ids=token_ids,
cumulative_logprob=None,
logprobs=logprobs,
finish_reason=finish_reason,
)
],
finished=finished,
metrics=None,
lora_request=None,
encoder_prompt=None,
encoder_prompt_token_ids=None,
num_cached_tokens=num_cached_tokens,
)
def _mock_engine() -> MagicMock:
engine = MagicMock(spec=AsyncLLM)
engine.errored = False
engine.model_config = MockModelConfig()
engine.vllm_config = MockVllmConfig(
engine.model_config, parallel_config=MockParallelConfig()
)
engine.input_processor = MagicMock()
engine.renderer = _build_renderer(engine.model_config)
return engine
def _parse_sse_chunks(chunks: list[str]) -> list[Any]:
"""Parse SSE chunks into dicts (JSON) or raw strings ([DONE])."""
parsed: list[Any] = []
for chunk in chunks:
assert chunk.startswith("data: ") and chunk.endswith("\n\n")
payload = chunk[len("data: ") : -len("\n\n")]
if payload == "[DONE]":
parsed.append("[DONE]")
else:
parsed.append(json.loads(payload))
return parsed
@pytest.mark.asyncio
async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1", token_ids=[10], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=1),
model=MODEL_NAME,
stream=False,
)
response = await serving.serve_tokens(request)
assert isinstance(response, GenerateResponse)
assert (
serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"]
is True
)
@pytest.mark.asyncio
async def test_stream_basic():
"""Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10])
yield _make_request_output("req-1", token_ids=[20, 30])
yield _make_request_output(
"req-1", token_ids=[40], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
)
response = await serving.serve_tokens(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
parsed = _parse_sse_chunks(chunks)
# 3 data chunks + [DONE]
assert parsed[-1] == "[DONE]"
data_chunks = [c for c in parsed if c != "[DONE]"]
assert len(data_chunks) == 3
assert data_chunks[0]["choices"][0]["token_ids"] == [10]
assert data_chunks[1]["choices"][0]["token_ids"] == [20, 30]
assert data_chunks[2]["choices"][0]["token_ids"] == [40]
assert data_chunks[2]["choices"][0]["finish_reason"] == "stop"
@pytest.mark.asyncio
async def test_stream_error_mid_generation():
"""finish_reason='error' mid-stream yields error chunk then [DONE]."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10])
yield _make_request_output(
"req-1", token_ids=[20], finish_reason="error", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
)
response = await serving.serve_tokens(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
assert len(chunks) >= 2
assert any("Internal server error" in chunk for chunk in chunks), (
f"Expected error message in chunks: {chunks}"
)
assert chunks[-1] == "data: [DONE]\n\n"
@pytest.mark.asyncio
async def test_stream_error_with_empty_delta():
"""finish_reason='error' with empty delta_token_ids still raises."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10])
yield _make_request_output(
"req-1", token_ids=[], finish_reason="error", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
)
response = await serving.serve_tokens(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
assert any("Internal server error" in chunk for chunk in chunks), (
f"Expected error message in chunks: {chunks}"
)
assert chunks[-1] == "data: [DONE]\n\n"
@pytest.mark.asyncio
async def test_stream_skips_empty_token_output():
"""Outputs with empty token_ids are skipped (no chunk emitted)."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10])
yield _make_request_output("req-1", token_ids=[])
yield _make_request_output(
"req-1", token_ids=[20], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
)
response = await serving.serve_tokens(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
parsed = _parse_sse_chunks(chunks)
assert parsed[-1] == "[DONE]"
data_chunks = [c for c in parsed if c != "[DONE]"]
# Only 2 data chunks — the empty one is skipped
assert len(data_chunks) == 2
assert data_chunks[0]["choices"][0]["token_ids"] == [10]
assert data_chunks[1]["choices"][0]["token_ids"] == [20]
@pytest.mark.asyncio
async def test_stream_include_usage():
"""stream_options.include_usage emits a final usage-only chunk."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10])
yield _make_request_output(
"req-1", token_ids=[20], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
stream_options=StreamOptions(include_usage=True),
)
response = await serving.serve_tokens(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
parsed = _parse_sse_chunks(chunks)
assert parsed[-1] == "[DONE]"
# The chunk before [DONE] should be the usage-only chunk
usage_chunk = parsed[-2]
assert usage_chunk["choices"] == []
assert usage_chunk["usage"]["prompt_tokens"] == 3
assert usage_chunk["usage"]["completion_tokens"] == 2
assert usage_chunk["usage"]["total_tokens"] == 5
@pytest.mark.asyncio
async def test_stream_continuous_usage():
"""continuous_usage_stats adds usage to every data chunk."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output("req-1", token_ids=[10])
yield _make_request_output(
"req-1", token_ids=[20], finish_reason="stop", finished=True
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
stream_options=StreamOptions(
include_usage=True,
continuous_usage_stats=True,
),
)
response = await serving.serve_tokens(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
parsed = _parse_sse_chunks(chunks)
data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
# Every data chunk should have usage
for i, dc in enumerate(data_chunks):
assert dc["usage"] is not None, f"chunk {i} missing usage"
assert dc["usage"]["prompt_tokens"] == 3
# First chunk: 1 completion token
assert data_chunks[0]["usage"]["completion_tokens"] == 1
assert data_chunks[0]["usage"]["total_tokens"] == 4
# Second chunk: 2 completion tokens (cumulative)
assert data_chunks[1]["usage"]["completion_tokens"] == 2
assert data_chunks[1]["usage"]["total_tokens"] == 5
@pytest.mark.asyncio
async def test_stream_with_logprobs():
"""Streaming with logprobs includes logprob data in each chunk."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10],
logprobs=[{10: Logprob(logprob=-0.5)}],
)
yield _make_request_output(
"req-1",
token_ids=[20],
logprobs=[{20: Logprob(logprob=-1.0)}],
finish_reason="stop",
finished=True,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10, logprobs=1),
model=MODEL_NAME,
stream=True,
)
response = await serving.serve_tokens(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
parsed = _parse_sse_chunks(chunks)
data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")]
for dc in data_chunks:
lp = dc["choices"][0]["logprobs"]
assert lp is not None
assert len(lp["content"]) == 1
assert lp["content"][0]["token"].startswith("token_id:")
@pytest.mark.asyncio
async def test_stream_prompt_tokens_details():
"""enable_prompt_tokens_details includes cached_tokens in final usage."""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10],
finish_reason="stop",
finished=True,
num_cached_tokens=2,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine, enable_prompt_tokens_details=True)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
stream_options=StreamOptions(include_usage=True),
)
response = await serving.serve_tokens(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
parsed = _parse_sse_chunks(chunks)
# Usage-only chunk (before [DONE])
usage_chunk = parsed[-2]
assert usage_chunk["choices"] == []
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 2
@pytest.mark.asyncio
async def test_stream_prompt_tokens_details_zero_cached():
"""enable_prompt_tokens_details includes cached_tokens=0 in final usage.
Regression test for https://github.com/vllm-project/vllm/issues/44377:
zero cached tokens must not be treated as falsy and omitted.
"""
engine = _mock_engine()
async def mock_generate(*args, **kwargs):
yield _make_request_output(
"req-1",
token_ids=[10],
finish_reason="stop",
finished=True,
num_cached_tokens=0,
)
engine.generate = MagicMock(side_effect=mock_generate)
serving = _build_serving_tokens(engine, enable_prompt_tokens_details=True)
request = GenerateRequest(
token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=10),
model=MODEL_NAME,
stream=True,
stream_options=StreamOptions(include_usage=True),
)
response = await serving.serve_tokens(request)
chunks = []
async for chunk in response:
chunks.append(chunk)
parsed = _parse_sse_chunks(chunks)
# Usage-only chunk (before [DONE])
usage_chunk = parsed[-2]
assert usage_chunk["choices"] == []
# Zero cached tokens must be present, not omitted
assert usage_chunk["usage"]["prompt_tokens_details"] is not None
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Roundtrip tests for multimodal serde used by the
token_in_token_out generate endpoint.
"""
import torch
from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
decode_mm_kwargs_item,
encode_mm_kwargs_item,
)
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.multimodal.inputs import (
MultiModalBatchedField,
MultiModalFieldElem,
MultiModalFlatField,
MultiModalKwargsItem,
MultiModalSharedField,
)
def test_mm_kwargs_item_roundtrip():
"""Full roundtrip test with all three field types and multiple dtypes."""
e1 = MultiModalFieldElem(
data=torch.zeros(1000, dtype=torch.bfloat16),
field=MultiModalBatchedField(),
)
e2 = MultiModalFieldElem(
data=torch.ones(100, dtype=torch.int32),
field=MultiModalSharedField(batch_size=4),
)
e3 = MultiModalFieldElem(
data=torch.randn(20, dtype=torch.float32),
field=MultiModalFlatField(slices=[slice(0, 10), slice(10, 20)], dim=0),
)
item = MultiModalKwargsItem({"pixel_values": e1, "grid_thw": e2, "embeds": e3})
encoded = encode_mm_kwargs_item(item)
# Encoded result is a base64 string
assert isinstance(encoded, str)
decoded = decode_mm_kwargs_item(encoded)
assert set(decoded.keys()) == {"pixel_values", "grid_thw", "embeds"}
assert torch.equal(item["pixel_values"].data, decoded["pixel_values"].data)
assert torch.equal(item["grid_thw"].data, decoded["grid_thw"].data)
assert torch.equal(item["embeds"].data, decoded["embeds"].data)
assert isinstance(decoded["pixel_values"].field, MultiModalBatchedField)
assert isinstance(decoded["grid_thw"].field, MultiModalSharedField)
assert isinstance(decoded["embeds"].field, MultiModalFlatField)
def test_mm_kwargs_item_none_data():
"""Roundtrip with None data field."""
elem = MultiModalFieldElem(
data=None,
field=MultiModalSharedField(batch_size=2),
)
item = MultiModalKwargsItem({"empty": elem})
encoded = encode_mm_kwargs_item(item)
decoded = decode_mm_kwargs_item(encoded)
assert decoded["empty"].data is None
assert isinstance(decoded["empty"].field, MultiModalSharedField)
def test_mm_kwargs_item_nested_tensors():
"""Roundtrip with nested tensor data."""
nested = [torch.randn(3, 4), torch.randn(5, 4)]
elem = MultiModalFieldElem(
data=nested,
field=MultiModalBatchedField(),
)
item = MultiModalKwargsItem({"nested": elem})
encoded = encode_mm_kwargs_item(item)
decoded = decode_mm_kwargs_item(encoded)
decoded_data = decoded["nested"].data
assert len(decoded_data) == 2
assert torch.equal(nested[0], decoded_data[0])
assert torch.equal(nested[1], decoded_data[1])
def test_mm_features_with_kwargs_data():
"""Test that MultiModalFeatures can carry serialized tensor data."""
elem = MultiModalFieldElem(
data=torch.randn(5, 3, dtype=torch.float32),
field=MultiModalBatchedField(),
)
item = MultiModalKwargsItem({"pixel_values": elem})
encoded = encode_mm_kwargs_item(item)
features = MultiModalFeatures(
mm_hashes={"image": ["abc123"]},
mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=10)]},
kwargs_data={"image": [encoded]},
)
# JSON roundtrip
json_str = features.model_dump_json()
features2 = MultiModalFeatures.model_validate_json(json_str)
assert features2.mm_hashes == {"image": ["abc123"]}
assert features2.kwargs_data is not None
assert len(features2.kwargs_data["image"]) == 1
decoded = decode_mm_kwargs_item(features2.kwargs_data["image"][0])
assert torch.equal(elem.data, decoded["pixel_values"].data)
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the token_in_token_out request/response protocol.
These tests intentionally avoid spinning up a server — they exercise the
pydantic validators on ``GenerateRequest`` directly so they run fast and
fail loudly if the validator semantics ever drift.
"""
import json
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
from vllm.sampling_params import SamplingParams
def _base_payload() -> dict:
return {"token_ids": [1, 2, 3], "sampling_params": {}}
def test_omitted_max_tokens_is_not_provided():
"""Body without ``max_tokens`` must surface as 'not provided' so the
server can apply its own default instead of the dataclass 16."""
req = GenerateRequest.model_validate(_base_payload())
# SamplingParams' dataclass default leaks through the parsed instance —
# this is exactly the bug the server-side defaulting works around.
assert req.sampling_params.max_tokens == 16
assert not req.is_sampling_param_provided("max_tokens")
def test_explicit_max_tokens_is_provided():
"""Even when the client picks the same value as the dataclass default,
it must register as explicitly set so the server won't override it."""
payload = _base_payload()
payload["sampling_params"] = {"max_tokens": 16}
req = GenerateRequest.model_validate(payload)
assert req.sampling_params.max_tokens == 16
assert req.is_sampling_param_provided("max_tokens")
payload["sampling_params"] = {"max_tokens": 256}
req = GenerateRequest.model_validate(payload)
assert req.sampling_params.max_tokens == 256
assert req.is_sampling_param_provided("max_tokens")
def test_other_fields_tracked_independently():
payload = _base_payload()
payload["sampling_params"] = {"temperature": 0.0}
req = GenerateRequest.model_validate(payload)
assert not req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
def test_json_roundtrip_preserves_provided_keys():
payload = _base_payload()
payload["sampling_params"] = {"temperature": 0.5}
req = GenerateRequest.model_validate_json(json.dumps(payload))
assert not req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
def test_internal_instance_construction_treats_all_as_provided():
"""When internal callers build ``GenerateRequest`` from a pre-resolved
``SamplingParams`` instance, every field is considered explicitly set
so server-side defaulting can't clobber values resolved upstream."""
sp = SamplingParams(max_tokens=500, temperature=0.0)
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
assert req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
# And keys we never touched should also count as provided in this path.
assert req.is_sampling_param_provided("top_p")
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import httpx
import numpy as np
import pybase64 as base64
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "TitanML/tiny-mixtral"
GEN_ENDPOINT = "/inference/v1/generate"
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
# The published config has sliding_window=4096, which produces
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
# FullAttentionSpec group, so we override sliding_window=null below.
NUM_LOCAL_EXPERTS = 8
NUM_EXPERTS_PER_TOK = 2
NUM_HIDDEN_LAYERS = 2
@pytest.fixture(scope="module")
def server():
args = [
"--max-model-len",
"256",
"--max-num-seqs",
"32",
"--enforce-eager",
"--enable-return-routed-experts",
"--hf-overrides",
'{"sliding_window": null}',
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_generate_routed_experts(client):
"""Test that /inference/v1/generate returns routed_experts when enabled."""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
choice = data["choices"][0]
assert choice["routed_experts"] is not None
assert choice["token_ids"] is not None
# routed_experts is base64-encoded .npy bytes; decode to ndarray.
routed_experts = np.load(io.BytesIO(base64.b64decode(choice["routed_experts"])))
assert routed_experts.ndim == 3
num_tokens, num_layers, topk = routed_experts.shape
assert num_tokens > 0
assert num_layers == NUM_HIDDEN_LAYERS
assert topk == NUM_EXPERTS_PER_TOK
assert (routed_experts >= 0).all()
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
@@ -0,0 +1,158 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for multimodal features through the /inference/v1/generate endpoint.
Mirrors test_serving_tokens.py but exercises the multimodal piping
using Qwen/Qwen3-VL-2B-Instruct end-to-end via the server's /render ->
/generate -> /detokenize path. Intentionally avoids running the HF
processor in the pytest parent process to keep os.fork() in sibling
tests (e.g. test_weight_transfer_llm.py) deadlock-free.
"""
import os
import httpx
import pytest
import pytest_asyncio
from PIL import Image
from tests.utils import RemoteOpenAIServer
from vllm.multimodal.utils import encode_image_url
MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct"
GEN_ENDPOINT = "/inference/v1/generate"
RENDER_ENDPOINT = "/v1/chat/completions/render"
DETOKENIZE_ENDPOINT = "/detokenize"
@pytest.fixture(scope="module")
def test_image():
return Image.new("RGB", (224, 224), color=(255, 0, 0))
@pytest.fixture(scope="module")
def server():
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"4096",
"--enforce-eager",
"--no-enable-prefix-caching",
]
envs = os.environ.copy()
envs["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_render_to_generate_roundtrip(client, test_image):
"""End-to-end: render a multimodal chat -> feed into generate -> decode.
All preprocessing and detokenization happens in the server subprocess;
the pytest parent never imports transformers or touches torch tensors.
"""
data_url = encode_image_url(test_image, format="PNG")
render_payload = {
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{
"type": "text",
"text": "What color is this image? Answer in one word.",
},
],
}
],
}
render_resp = await client.post(RENDER_ENDPOINT, json=render_payload)
render_resp.raise_for_status()
render_data = render_resp.json()
# Validate render output structure: keys exist and values are non-empty
# and well-typed.
assert "token_ids" in render_data
assert isinstance(render_data["token_ids"], list)
assert len(render_data["token_ids"]) > 0
assert all(isinstance(t, int) for t in render_data["token_ids"])
assert "features" in render_data
features = render_data["features"]
assert features is not None
assert isinstance(features, dict)
assert "mm_hashes" in features
assert "image" in features["mm_hashes"]
image_hashes = features["mm_hashes"]["image"]
assert isinstance(image_hashes, list)
assert len(image_hashes) > 0
assert all(isinstance(h, str) and h for h in image_hashes)
assert "mm_placeholders" in features
assert "image" in features["mm_placeholders"]
image_placeholders = features["mm_placeholders"]["image"]
assert isinstance(image_placeholders, list)
assert len(image_placeholders) > 0
for p in image_placeholders:
assert isinstance(p.get("offset"), int)
assert isinstance(p.get("length"), int)
assert p["length"] > 0
assert "kwargs_data" in features
assert "image" in features["kwargs_data"]
assert len(features["kwargs_data"]["image"]) > 0
# Build generate request from render output
generate_payload = render_data
generate_payload["sampling_params"] = {
"max_tokens": 10,
"temperature": 0.0,
}
gen_resp = await client.post(GEN_ENDPOINT, json=generate_payload)
gen_resp.raise_for_status()
gen_data = gen_resp.json()
assert "choices" in gen_data
assert isinstance(gen_data["choices"], list)
assert len(gen_data["choices"]) >= 1
choice = gen_data["choices"][0]
assert "token_ids" in choice
assert isinstance(choice["token_ids"], list)
assert len(choice["token_ids"]) > 0
assert all(isinstance(t, int) for t in choice["token_ids"])
detok_resp = await client.post(
DETOKENIZE_ENDPOINT,
json={"model": MODEL_NAME, "tokens": choice["token_ids"]},
)
detok_resp.raise_for_status()
detok_data = detok_resp.json()
assert "prompt" in detok_data
text = detok_data["prompt"]
assert isinstance(text, str)
assert len(text) > 0
assert "red" in text.lower(), (
f"Expected model to identify the red image, got: {text!r}"
)
@@ -0,0 +1,425 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
import httpx
import pytest
import pytest_asyncio
from transformers import AutoTokenizer
from tests.utils import RemoteOpenAIServer
from vllm.config import ModelConfig
from vllm.config.utils import getattr_iter
from vllm.v1.engine.detokenizer import check_stop_strings
MODEL_NAME = "Qwen/Qwen3-0.6B"
GEN_ENDPOINT = "/inference/v1/generate"
def get_vocab_size(model_name):
config = ModelConfig(
model=model_name,
seed=0,
dtype="bfloat16",
)
return config.get_vocab_size()
@pytest.fixture(scope="module")
def tokenizer():
return AutoTokenizer.from_pretrained(MODEL_NAME)
@pytest.fixture(scope="module")
def messages():
return [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "How many countries are in the EU?"},
]
@pytest.fixture(scope="module")
def server(request):
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"1024",
"--enforce-eager",
# On ROCm (e.g. MI355X/gfx950), bf16 GEMM results can differ by
# 1 ULP when the batch dimension (M) changes, because different M
# values cause the Tensile backend to select different tile
# configurations with different fp32 accumulation orders. With
# prefix caching, cache-miss prefills compute all tokens in one
# pass (large M) while cache-hit requests compute only the
# uncached suffix (small M), seeding a divergence that amplifies
# through the residual stream and flips argmax tokens.
# See: https://github.com/vllm-project/vllm/issues/33123
#
# Either disable prefix caching entirely, or enable it with
# --deterministic-prefix-caching which forces cache-miss prefills
# to split at block boundaries so the suffix GEMM shape is always
# identical regardless of cache state.
#
# Option A: disable prefix caching
"--no-enable-prefix-caching",
#
# Option B: deterministic prefix caching
# "--enable-prefix-caching",
# "--deterministic-prefix-caching",
]
extra_args = getattr(request, "param", None)
if extra_args is not None:
args = args + (
list(extra_args)
if isinstance(extra_args, (list, tuple))
else [str(extra_args)]
)
envs = os.environ.copy()
# See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3888060787
envs["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=envs) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_generate_endpoint(client):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert "choices" in data
@pytest.mark.asyncio
async def test_generate_defaults_max_tokens_when_omitted(client):
"""Regression: omitting ``max_tokens`` must not silently cap at 16.
``SamplingParams.max_tokens`` defaults to 16. Before the server-side
defaulting was wired up, every request that didn't set ``max_tokens``
truncated mid-generation. The server should now fill it in from
``max_model_len - prompt_len`` (matching ``/v1/chat/completions``).
"""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"temperature": 0.0,
"ignore_eos": True,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
completion_tokens = len(data["choices"][0]["token_ids"])
# max_model_len=1024 in the test fixture, prompt is 3 tokens, so we
# should get ~1021 tokens of output (capped at max_model_len boundary).
assert completion_tokens > 16, (
f"expected server-side default to exceed the legacy 16-token cap, "
f"got {completion_tokens}"
)
@pytest.mark.asyncio
async def test_generate_stream(client):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": True,
}
async with client.stream("POST", GEN_ENDPOINT, json=payload) as resp:
resp.raise_for_status()
chunks = []
async for line in resp.aiter_lines():
if not line.startswith("data: "):
continue
payload_str = line[len("data: ") :]
if payload_str == "[DONE]":
break
chunks.append(json.loads(payload_str))
assert len(chunks) > 0
# Every chunk has choices with token_ids
all_token_ids = []
for chunk in chunks:
assert "choices" in chunk
assert len(chunk["choices"]) == 1
choice = chunk["choices"][0]
assert "token_ids" in choice
assert len(choice["token_ids"]) > 0
all_token_ids.extend(choice["token_ids"])
# Last chunk should have a finish_reason
assert chunks[-1]["choices"][0]["finish_reason"] is not None
# Streaming should produce the same tokens as non-streaming
non_stream_resp = await client.post(
GEN_ENDPOINT,
json={
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5, "temperature": 0.0},
"stream": False,
},
)
non_stream_data = non_stream_resp.json()
# Just verify we got the right number of tokens
assert len(all_token_ids) == len(non_stream_data["choices"][0]["token_ids"])
@pytest.mark.asyncio
@pytest.mark.parametrize("logprobs_value", [0, 1, 5])
async def test_generate_logprobs(client, logprobs_value):
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"max_tokens": 5,
"temperature": 0.0,
"logprobs": logprobs_value,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
choice = data["choices"][0]
assert choice["logprobs"] is not None
logprobs_content = choice["logprobs"]["content"]
assert len(logprobs_content) == len(choice["token_ids"])
for entry in logprobs_content:
assert "logprob" in entry
assert len(entry["top_logprobs"]) >= 1
assert len(entry["top_logprobs"]) == max(logprobs_value, 1)
@pytest.mark.asyncio
async def test_same_response_as_chat_completions(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
for ignore_eos in [True, False]:
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
# NOTE coordinator will set this to skip detokenization
"detokenize": False,
"ignore_eos": ignore_eos,
},
"stream": False,
}
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_data = generate_resp.json()
gen_token_ids = generate_data["choices"][0]["token_ids"]
generate_res = tokenizer.decode(gen_token_ids, skip_special_tokens=True)
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"ignore_eos": ignore_eos,
"chat_template_kwargs": {"enable_thinking": False},
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
if ignore_eos:
# When ignoring EOS, only compare up to the first EOS token
# Post-EOS generation is undefined and may differ
eos_tokens = {
tokenizer.eos_token_id,
*getattr_iter(
tokenizer,
[
"extra_special_tokens_ids", # Transformers v5
"additional_special_tokens_ids", # Transformers v4
],
[],
),
}
# Find first EOS in generated tokens
eos_pos = None
for i, tid in enumerate(gen_token_ids):
if tid in eos_tokens:
eos_pos = i
break
if eos_pos is not None:
gen_token_ids_truncated = gen_token_ids[:eos_pos]
generate_res = tokenizer.decode(
gen_token_ids_truncated, skip_special_tokens=True
)
# Truncate completions_res to same length for comparison
completions_res = completions_res[: len(generate_res)]
assert generate_res == completions_res
@pytest.mark.asyncio
async def test_stop_string_workflow(client, tokenizer, messages):
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
payload = {
"model": MODEL_NAME,
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
"detokenize": False,
# stop strings are only supported when detokenize is True.
"stop": ["27 member"],
},
# TODO stream test is much more interesting
"stream": False,
}
with pytest.raises(httpx.HTTPStatusError):
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_resp.raise_for_status()
payload["sampling_params"]["stop"] = None
generate_resp = await client.post(
GEN_ENDPOINT, json=payload, headers={"X-Request-Id": "42"}
)
generate_data = generate_resp.json()
generate_res = tokenizer.decode(
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
)
# NOTE This is under the responsibility of the coordinator
# stop_checker = StopChecker(
# max_model_len=1024, get_tokenizer_for_seq=lambda _: tokenizer
# )
stop_str, truncate_to = check_stop_strings(
generate_res, len(generate_res), ["27 member"], False
)
assert stop_str == "27 member"
# abort request that hit stop string (requires tokens-only mode)
# res = await client.post("/abort_requests", json={"request_ids": ["generate-tokens-42"]}) # noqa: E501
# res.raise_for_status()
generate_res = generate_res[:truncate_to]
# Get stop_str response from chat completions
payload = {
"model": MODEL_NAME,
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"stop": ["27 member"],
"chat_template_kwargs": dict(enable_thinking=False),
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
assert generate_res == completions_res
@pytest.mark.asyncio
@pytest.mark.parametrize(
"server",
[
[
"--enable-lora",
"--lora-modules",
"Alice=charent/self_cognition_Alice",
"Bob=charent/self_cognition_Bob",
"--max-lora-rank",
"64",
"--max-cpu-loras",
"2",
]
],
indirect=True,
)
async def test_generate_with_lora_adapter(client, tokenizer, messages):
# Verify adapters are listed
models_resp = await client.get("/v1/models")
models_resp.raise_for_status()
models = {m["id"] for m in models_resp.json().get("data", [])}
assert {"Alice", "Bob"}.issubset(models)
# Generate using a LoRA adapter by specifying its name as the model
payload = {
"model": "Alice",
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 5},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
assert "choices" in data
token_ids = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
enable_thinking=False, # default with Qwen3
return_dict=True, # default with Transformers v5
).input_ids
payload = {
"model": "Alice",
"token_ids": token_ids,
"sampling_params": {
"max_tokens": 24,
"temperature": 0.0,
"detokenize": False,
},
"stream": False,
}
generate_resp = await client.post(GEN_ENDPOINT, json=payload)
generate_data = generate_resp.json()
generate_res = tokenizer.decode(
generate_data["choices"][0]["token_ids"], skip_special_tokens=True
)
payload = {
"model": "Alice",
"messages": messages,
"max_tokens": 24,
"temperature": 0.0,
"stream": False,
"chat_template_kwargs": dict(enable_thinking=False),
}
completions_resp = await client.post("/v1/chat/completions", json=payload)
completions_data = completions_resp.json()
completions_res = completions_data["choices"][0]["message"]["content"]
assert generate_res == completions_res
@@ -0,0 +1,28 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
from vllm.logprobs import Logprob
def test_top_logprobs_alternatives_have_own_token_ids():
"""Each top_logprobs alternative must carry its own token_id placeholder."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[262],
top_logprobs=[{262: Logprob(-0.1), 257: Logprob(-1.2), 428: Logprob(-2.3)}],
num_output_top_logprobs=2,
)
tokens = {e.token for e in result.content[0].top_logprobs}
assert tokens == {"token_id:262", "token_id:257"}, f"got {tokens}"
def test_logprobs_zero_emits_sampled_token():
"""logprobs=0 must still emit 1 entry (the sampled token)."""
result = ServingTokens._create_tokens_logprobs(
None,
token_ids=[7],
top_logprobs=[{7: Logprob(-0.9), 8: Logprob(-1.1)}],
num_output_top_logprobs=0,
)
assert len(result.content[0].top_logprobs) == 1