[Frontend] Consolidate online serving utils. (#44479)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
get_max_tokens,
|
||||
sanitize_message,
|
||||
should_include_usage,
|
||||
)
|
||||
|
||||
|
||||
def test_sanitize_message():
|
||||
assert (
|
||||
sanitize_message("<_io.BytesIO object at 0x7a95e299e750>")
|
||||
== "<_io.BytesIO object>"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("stream_options", "expected"),
|
||||
[
|
||||
(None, (True, True)),
|
||||
(StreamOptions(include_usage=False), (True, True)),
|
||||
(
|
||||
StreamOptions(include_usage=False, continuous_usage_stats=False),
|
||||
(True, True),
|
||||
),
|
||||
(
|
||||
StreamOptions(include_usage=True, continuous_usage_stats=False),
|
||||
(True, True),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_should_include_usage_force_enables_continuous_usage(stream_options, expected):
|
||||
assert should_include_usage(stream_options, True) == expected
|
||||
|
||||
|
||||
class TestGetMaxTokens:
|
||||
"""Tests for get_max_tokens() to ensure generation_config's max_tokens
|
||||
acts as a default when from model author, and as a ceiling when
|
||||
explicitly set by the user."""
|
||||
|
||||
def test_default_sampling_params_used_when_no_request_max_tokens(self):
|
||||
"""When user doesn't specify max_tokens, generation_config default
|
||||
should apply."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=24000,
|
||||
max_tokens=None,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
)
|
||||
assert result == 2048
|
||||
|
||||
def test_request_max_tokens_not_capped_by_default_sampling_params(self):
|
||||
"""When user specifies max_tokens in request, model author's
|
||||
generation_config max_tokens must NOT cap it (fixes #34005)."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=24000,
|
||||
max_tokens=5000,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
)
|
||||
assert result == 5000
|
||||
|
||||
def test_override_max_tokens_caps_request(self):
|
||||
"""When user explicitly sets max_tokens, it acts as a ceiling."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=24000,
|
||||
max_tokens=5000,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
override_max_tokens=2048,
|
||||
)
|
||||
assert result == 2048
|
||||
|
||||
def test_override_max_tokens_used_as_default(self):
|
||||
"""When no request max_tokens, override still applies as default."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=24000,
|
||||
max_tokens=None,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
override_max_tokens=2048,
|
||||
)
|
||||
assert result == 2048
|
||||
|
||||
def test_max_model_len_still_caps_output(self):
|
||||
"""max_model_len - input_length is always the hard ceiling."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=3000,
|
||||
max_tokens=5000,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
)
|
||||
assert result == 2900 # 3000 - 100
|
||||
|
||||
def test_request_max_tokens_smaller_than_default(self):
|
||||
"""When user explicitly requests fewer tokens than gen_config default,
|
||||
that should be respected."""
|
||||
result = get_max_tokens(
|
||||
max_model_len=24000,
|
||||
max_tokens=512,
|
||||
input_length=100,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
)
|
||||
assert result == 512
|
||||
|
||||
def test_input_length_exceeds_max_model_len(self):
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Input length .* exceeds model's maximum context length .*",
|
||||
):
|
||||
get_max_tokens(
|
||||
max_model_len=100,
|
||||
max_tokens=50,
|
||||
input_length=150,
|
||||
default_sampling_params={"max_tokens": 2048},
|
||||
)
|
||||
@@ -0,0 +1,76 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ``system_fingerprint`` construction."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.serve.utils import fingerprint as fp
|
||||
|
||||
|
||||
def _cfg(tp=1, pp=1, dp=1, ep=False, digest="a3b21f94deadbeef"):
|
||||
c = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=tp,
|
||||
pipeline_parallel_size=pp,
|
||||
data_parallel_size=dp,
|
||||
enable_expert_parallel=ep,
|
||||
)
|
||||
)
|
||||
c.compute_hash = lambda: digest # type: ignore[attr-defined]
|
||||
return c
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset():
|
||||
fp.set_default_fingerprint_mode("full")
|
||||
yield
|
||||
fp.set_default_fingerprint_mode("full")
|
||||
|
||||
|
||||
def test_four_modes_produce_expected_shapes():
|
||||
from vllm import __version__ as v
|
||||
|
||||
cfg = _cfg(tp=8, ep=True)
|
||||
|
||||
assert fp.build_system_fingerprint(cfg, "full") == (f"vllm-{v}-tp8-ep-a3b21f94")
|
||||
assert fp.build_system_fingerprint(cfg, "hash") == f"vllm-{v}-a3b21f94"
|
||||
assert fp.build_system_fingerprint(cfg, "custom", "my-fp") == "my-fp"
|
||||
assert fp.build_system_fingerprint(cfg, "none") is None
|
||||
|
||||
|
||||
def test_full_mode_emits_only_non_trivial_parallelism():
|
||||
from vllm import __version__ as v
|
||||
|
||||
# Single-GPU: nothing between version and hash.
|
||||
assert fp.build_system_fingerprint(_cfg(), "full") == f"vllm-{v}-a3b21f94"
|
||||
# All parallelism axes.
|
||||
assert (
|
||||
fp.build_system_fingerprint(_cfg(tp=8, pp=2, dp=4, ep=True), "full")
|
||||
== f"vllm-{v}-tp8-pp2-dp4-ep-a3b21f94"
|
||||
)
|
||||
|
||||
|
||||
def test_get_respects_set_default():
|
||||
cfg = _cfg(tp=8)
|
||||
full = fp.get_system_fingerprint(cfg)
|
||||
assert full == fp.get_system_fingerprint(cfg)
|
||||
|
||||
fp.set_default_fingerprint_mode("hash")
|
||||
hashed = fp.get_system_fingerprint(cfg)
|
||||
assert hashed != full
|
||||
assert "tp8" not in hashed
|
||||
|
||||
fp.set_default_fingerprint_mode("custom", "deploy-42")
|
||||
assert fp.get_system_fingerprint(cfg) == "deploy-42"
|
||||
|
||||
fp.set_default_fingerprint_mode("none")
|
||||
assert fp.get_system_fingerprint(cfg) is None
|
||||
|
||||
|
||||
def test_compute_hash_failure_does_not_raise():
|
||||
cfg = _cfg()
|
||||
cfg.compute_hash = lambda: (_ for _ in ()).throw(RuntimeError("boom"))
|
||||
assert fp.build_system_fingerprint(cfg, "full").endswith("-nohash")
|
||||
assert fp.build_system_fingerprint(cfg, "hash").endswith("-nohash")
|
||||
@@ -0,0 +1,248 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
|
||||
|
||||
def test_request_logger_log_outputs():
|
||||
"""Test the new log_outputs functionality."""
|
||||
# Create a mock logger to capture log calls
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
|
||||
request_logger = RequestLogger(max_log_len=None)
|
||||
|
||||
# Test basic output logging
|
||||
request_logger.log_outputs(
|
||||
request_id="test-123",
|
||||
outputs="Hello, world!",
|
||||
output_token_ids=[1, 2, 3, 4],
|
||||
finish_reason="stop",
|
||||
is_streaming=False,
|
||||
delta=False,
|
||||
)
|
||||
|
||||
mock_logger.info.assert_called_once()
|
||||
call_args = mock_logger.info.call_args.args
|
||||
assert "Generated response %s%s" in call_args[0]
|
||||
assert call_args[1] == "test-123"
|
||||
assert call_args[3] == "Hello, world!"
|
||||
assert call_args[4] == [1, 2, 3, 4]
|
||||
assert call_args[5] == "stop"
|
||||
|
||||
|
||||
def test_request_logger_log_outputs_streaming_delta():
|
||||
"""Test log_outputs with streaming delta mode."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
|
||||
request_logger = RequestLogger(max_log_len=None)
|
||||
|
||||
# Test streaming delta logging
|
||||
request_logger.log_outputs(
|
||||
request_id="test-456",
|
||||
outputs="Hello",
|
||||
output_token_ids=[1],
|
||||
finish_reason=None,
|
||||
is_streaming=True,
|
||||
delta=True,
|
||||
)
|
||||
|
||||
mock_logger.info.assert_called_once()
|
||||
call_args = mock_logger.info.call_args.args
|
||||
assert "Generated response %s%s" in call_args[0]
|
||||
assert call_args[1] == "test-456"
|
||||
assert call_args[2] == " (streaming delta)"
|
||||
assert call_args[3] == "Hello"
|
||||
assert call_args[4] == [1]
|
||||
assert call_args[5] is None
|
||||
|
||||
|
||||
def test_request_logger_log_outputs_streaming_complete():
|
||||
"""Test log_outputs with streaming complete mode."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
|
||||
request_logger = RequestLogger(max_log_len=None)
|
||||
|
||||
# Test streaming complete logging
|
||||
request_logger.log_outputs(
|
||||
request_id="test-789",
|
||||
outputs="Complete response",
|
||||
output_token_ids=[1, 2, 3],
|
||||
finish_reason="length",
|
||||
is_streaming=True,
|
||||
delta=False,
|
||||
)
|
||||
|
||||
mock_logger.info.assert_called_once()
|
||||
call_args = mock_logger.info.call_args.args
|
||||
assert "Generated response %s%s" in call_args[0]
|
||||
assert call_args[1] == "test-789"
|
||||
assert call_args[2] == " (streaming complete)"
|
||||
assert call_args[3] == "Complete response"
|
||||
assert call_args[4] == [1, 2, 3]
|
||||
assert call_args[5] == "length"
|
||||
|
||||
|
||||
def test_request_logger_log_outputs_with_truncation():
|
||||
"""Test log_outputs respects max_log_len setting."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
|
||||
# Set max_log_len to 10
|
||||
request_logger = RequestLogger(max_log_len=10)
|
||||
|
||||
# Test output truncation
|
||||
long_output = "This is a very long output that should be truncated"
|
||||
long_token_ids = list(range(20)) # 20 tokens
|
||||
|
||||
request_logger.log_outputs(
|
||||
request_id="test-truncate",
|
||||
outputs=long_output,
|
||||
output_token_ids=long_token_ids,
|
||||
finish_reason="stop",
|
||||
is_streaming=False,
|
||||
delta=False,
|
||||
)
|
||||
|
||||
mock_logger.info.assert_called_once()
|
||||
call_args = mock_logger.info.call_args
|
||||
|
||||
# Check that output was truncated to first 10 characters
|
||||
logged_output = call_args[0][3]
|
||||
assert logged_output == "This is a "
|
||||
assert len(logged_output) == 10
|
||||
|
||||
# Check that token IDs were truncated to first 10 tokens
|
||||
logged_token_ids = call_args[0][4]
|
||||
assert logged_token_ids == list(range(10))
|
||||
assert len(logged_token_ids) == 10
|
||||
|
||||
|
||||
def test_request_logger_log_outputs_none_values():
|
||||
"""Test log_outputs handles None values correctly."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
|
||||
request_logger = RequestLogger(max_log_len=None)
|
||||
|
||||
# Test with None output_token_ids
|
||||
request_logger.log_outputs(
|
||||
request_id="test-none",
|
||||
outputs="Test output",
|
||||
output_token_ids=None,
|
||||
finish_reason="stop",
|
||||
is_streaming=False,
|
||||
delta=False,
|
||||
)
|
||||
|
||||
mock_logger.info.assert_called_once()
|
||||
call_args = mock_logger.info.call_args.args
|
||||
assert "Generated response %s%s" in call_args[0]
|
||||
assert call_args[1] == "test-none"
|
||||
assert call_args[3] == "Test output"
|
||||
assert call_args[4] is None
|
||||
assert call_args[5] == "stop"
|
||||
|
||||
|
||||
def test_request_logger_log_outputs_empty_output():
|
||||
"""Test log_outputs handles empty output correctly."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
|
||||
request_logger = RequestLogger(max_log_len=5)
|
||||
|
||||
# Test with empty output
|
||||
request_logger.log_outputs(
|
||||
request_id="test-empty",
|
||||
outputs="",
|
||||
output_token_ids=[],
|
||||
finish_reason="stop",
|
||||
is_streaming=False,
|
||||
delta=False,
|
||||
)
|
||||
|
||||
mock_logger.info.assert_called_once()
|
||||
call_args = mock_logger.info.call_args.args
|
||||
assert "Generated response %s%s" in call_args[0]
|
||||
assert call_args[1] == "test-empty"
|
||||
assert call_args[3] == ""
|
||||
assert call_args[4] == []
|
||||
assert call_args[5] == "stop"
|
||||
|
||||
|
||||
def test_request_logger_log_outputs_integration():
|
||||
"""Test that log_outputs can be called alongside log_inputs."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
|
||||
request_logger = RequestLogger(max_log_len=None)
|
||||
|
||||
# Test that both methods can be called without interference
|
||||
request_logger.log_inputs(
|
||||
request_id="test-integration",
|
||||
prompt="Test prompt",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_embeds=None,
|
||||
params=None,
|
||||
lora_request=None,
|
||||
)
|
||||
|
||||
request_logger.log_outputs(
|
||||
request_id="test-integration",
|
||||
outputs="Test output",
|
||||
output_token_ids=[4, 5, 6],
|
||||
finish_reason="stop",
|
||||
is_streaming=False,
|
||||
delta=False,
|
||||
)
|
||||
|
||||
# Should have been called twice - once for inputs, once for outputs
|
||||
assert mock_logger.info.call_count == 2
|
||||
|
||||
# Check that the calls were made with correct patterns
|
||||
input_call = mock_logger.info.call_args_list[0][0]
|
||||
output_call = mock_logger.info.call_args_list[1][0]
|
||||
|
||||
assert "Received request %s" in input_call[0]
|
||||
assert input_call[1] == "test-integration"
|
||||
|
||||
assert "Generated response %s%s" in output_call[0]
|
||||
assert output_call[1] == "test-integration"
|
||||
|
||||
|
||||
def test_streaming_complete_logs_full_text_content():
|
||||
"""Test that streaming complete logging includes
|
||||
full accumulated text, not just token count."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("vllm.entrypoints.serve.utils.request_logger.logger", mock_logger):
|
||||
request_logger = RequestLogger(max_log_len=None)
|
||||
|
||||
# Test with actual content instead of token count format
|
||||
full_response = "This is a complete response from streaming"
|
||||
request_logger.log_outputs(
|
||||
request_id="test-streaming-full-text",
|
||||
outputs=full_response,
|
||||
output_token_ids=None,
|
||||
finish_reason="streaming_complete",
|
||||
is_streaming=True,
|
||||
delta=False,
|
||||
)
|
||||
|
||||
mock_logger.info.assert_called_once()
|
||||
call_args = mock_logger.info.call_args.args
|
||||
|
||||
# Verify the logged output is the full text, not a token count format
|
||||
logged_output = call_args[3]
|
||||
assert logged_output == full_response
|
||||
assert "tokens>" not in logged_output
|
||||
assert "streaming_complete" not in logged_output
|
||||
|
||||
# Verify other parameters
|
||||
assert call_args[1] == "test-streaming-full-text"
|
||||
assert call_args[2] == " (streaming complete)"
|
||||
assert call_args[5] == "streaming_complete"
|
||||
@@ -0,0 +1,72 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from ssl import SSLContext
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.serve.utils.ssl import SSLCertRefresher
|
||||
|
||||
|
||||
class MockSSLContext(SSLContext):
|
||||
def __init__(self):
|
||||
self.load_cert_chain_count = 0
|
||||
self.load_ca_count = 0
|
||||
|
||||
def load_cert_chain(
|
||||
self,
|
||||
certfile,
|
||||
keyfile=None,
|
||||
password=None,
|
||||
):
|
||||
self.load_cert_chain_count += 1
|
||||
|
||||
def load_verify_locations(
|
||||
self,
|
||||
cafile=None,
|
||||
capath=None,
|
||||
cadata=None,
|
||||
):
|
||||
self.load_ca_count += 1
|
||||
|
||||
|
||||
def create_file() -> str:
|
||||
with tempfile.NamedTemporaryFile(dir="/tmp", delete=False) as f:
|
||||
return f.name
|
||||
|
||||
|
||||
def touch_file(path: str) -> None:
|
||||
Path(path).touch()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ssl_refresher():
|
||||
ssl_context = MockSSLContext()
|
||||
key_path = create_file()
|
||||
cert_path = create_file()
|
||||
ca_path = create_file()
|
||||
ssl_refresher = SSLCertRefresher(ssl_context, key_path, cert_path, ca_path)
|
||||
await asyncio.sleep(1)
|
||||
assert ssl_context.load_cert_chain_count == 0
|
||||
assert ssl_context.load_ca_count == 0
|
||||
|
||||
touch_file(key_path)
|
||||
await asyncio.sleep(1)
|
||||
assert ssl_context.load_cert_chain_count == 1
|
||||
assert ssl_context.load_ca_count == 0
|
||||
|
||||
touch_file(cert_path)
|
||||
touch_file(ca_path)
|
||||
await asyncio.sleep(1)
|
||||
assert ssl_context.load_cert_chain_count == 2
|
||||
assert ssl_context.load_ca_count == 1
|
||||
|
||||
ssl_refresher.stop()
|
||||
|
||||
touch_file(cert_path)
|
||||
touch_file(ca_path)
|
||||
await asyncio.sleep(1)
|
||||
assert ssl_context.load_cert_chain_count == 2
|
||||
assert ssl_context.load_ca_count == 1
|
||||
Reference in New Issue
Block a user