Add system_fingerprint field to OpenAI-compatible API responses (#40537)
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,76 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Unit tests for ``system_fingerprint`` construction."""
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.entrypoints.openai import fingerprint as fp
|
||||||
|
|
||||||
|
|
||||||
|
def _cfg(tp=1, pp=1, dp=1, ep=False, digest="a3b21f94deadbeef"):
|
||||||
|
c = SimpleNamespace(
|
||||||
|
parallel_config=SimpleNamespace(
|
||||||
|
tensor_parallel_size=tp,
|
||||||
|
pipeline_parallel_size=pp,
|
||||||
|
data_parallel_size=dp,
|
||||||
|
enable_expert_parallel=ep,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
c.compute_hash = lambda: digest # type: ignore[attr-defined]
|
||||||
|
return c
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset():
|
||||||
|
fp.set_default_fingerprint_mode("full")
|
||||||
|
yield
|
||||||
|
fp.set_default_fingerprint_mode("full")
|
||||||
|
|
||||||
|
|
||||||
|
def test_four_modes_produce_expected_shapes():
|
||||||
|
from vllm import __version__ as v
|
||||||
|
|
||||||
|
cfg = _cfg(tp=8, ep=True)
|
||||||
|
|
||||||
|
assert fp.build_system_fingerprint(cfg, "full") == (f"vllm-{v}-tp8-ep-a3b21f94")
|
||||||
|
assert fp.build_system_fingerprint(cfg, "hash") == f"vllm-{v}-a3b21f94"
|
||||||
|
assert fp.build_system_fingerprint(cfg, "custom", "my-fp") == "my-fp"
|
||||||
|
assert fp.build_system_fingerprint(cfg, "none") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_full_mode_emits_only_non_trivial_parallelism():
|
||||||
|
from vllm import __version__ as v
|
||||||
|
|
||||||
|
# Single-GPU: nothing between version and hash.
|
||||||
|
assert fp.build_system_fingerprint(_cfg(), "full") == f"vllm-{v}-a3b21f94"
|
||||||
|
# All parallelism axes.
|
||||||
|
assert (
|
||||||
|
fp.build_system_fingerprint(_cfg(tp=8, pp=2, dp=4, ep=True), "full")
|
||||||
|
== f"vllm-{v}-tp8-pp2-dp4-ep-a3b21f94"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_respects_set_default():
|
||||||
|
cfg = _cfg(tp=8)
|
||||||
|
full = fp.get_system_fingerprint(cfg)
|
||||||
|
assert full == fp.get_system_fingerprint(cfg)
|
||||||
|
|
||||||
|
fp.set_default_fingerprint_mode("hash")
|
||||||
|
hashed = fp.get_system_fingerprint(cfg)
|
||||||
|
assert hashed != full
|
||||||
|
assert "tp8" not in hashed
|
||||||
|
|
||||||
|
fp.set_default_fingerprint_mode("custom", "deploy-42")
|
||||||
|
assert fp.get_system_fingerprint(cfg) == "deploy-42"
|
||||||
|
|
||||||
|
fp.set_default_fingerprint_mode("none")
|
||||||
|
assert fp.get_system_fingerprint(cfg) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_compute_hash_failure_does_not_raise():
|
||||||
|
cfg = _cfg()
|
||||||
|
cfg.compute_hash = lambda: (_ for _ in ()).throw(RuntimeError("boom"))
|
||||||
|
assert fp.build_system_fingerprint(cfg, "full").endswith("-nohash")
|
||||||
|
assert fp.build_system_fingerprint(cfg, "hash").endswith("-nohash")
|
||||||
@@ -317,4 +317,5 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
|||||||
model=model_name,
|
model=model_name,
|
||||||
choices=choices,
|
choices=choices,
|
||||||
usage=usage,
|
usage=usage,
|
||||||
|
system_fingerprint=self.system_fingerprint,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -129,6 +129,9 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
|
|||||||
model: str
|
model: str
|
||||||
choices: list[ChatCompletionResponseStreamChoice]
|
choices: list[ChatCompletionResponseStreamChoice]
|
||||||
usage: UsageInfo | None = Field(default=None)
|
usage: UsageInfo | None = Field(default=None)
|
||||||
|
# Set only on the final chunk of a stream to mirror non-streaming responses
|
||||||
|
# without the per-chunk serialization overhead.
|
||||||
|
system_fingerprint: str | None = None
|
||||||
# not part of the OpenAI spec but for tracing the tokens
|
# not part of the OpenAI spec but for tracing the tokens
|
||||||
prompt_token_ids: list[int] | None = None
|
prompt_token_ids: list[int] | None = None
|
||||||
|
|
||||||
|
|||||||
@@ -1195,6 +1195,16 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
choices=[choice_data],
|
choices=[choice_data],
|
||||||
model=model_name,
|
model=model_name,
|
||||||
)
|
)
|
||||||
|
# Stamp the fingerprint on terminal chunks only (those with
|
||||||
|
# finish_reason set). When ``include_usage`` is on, the
|
||||||
|
# trailing usage chunk below overrides this as the true
|
||||||
|
# final message.
|
||||||
|
if (
|
||||||
|
not include_usage
|
||||||
|
and self.system_fingerprint is not None
|
||||||
|
and choice_data.finish_reason is not None
|
||||||
|
):
|
||||||
|
chunk.system_fingerprint = self.system_fingerprint
|
||||||
|
|
||||||
# handle usage stats if requested & if continuous
|
# handle usage stats if requested & if continuous
|
||||||
if include_continuous_usage:
|
if include_continuous_usage:
|
||||||
@@ -1229,6 +1239,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
choices=[],
|
choices=[],
|
||||||
model=model_name,
|
model=model_name,
|
||||||
usage=final_usage,
|
usage=final_usage,
|
||||||
|
system_fingerprint=self.system_fingerprint,
|
||||||
)
|
)
|
||||||
final_usage_data = final_usage_chunk.model_dump_json(
|
final_usage_data = final_usage_chunk.model_dump_json(
|
||||||
exclude_unset=True, exclude_none=True
|
exclude_unset=True, exclude_none=True
|
||||||
@@ -1637,6 +1648,7 @@ class OpenAIServingChat(OpenAIServing):
|
|||||||
model=model_name,
|
model=model_name,
|
||||||
choices=choices,
|
choices=choices,
|
||||||
usage=usage,
|
usage=usage,
|
||||||
|
system_fingerprint=self.system_fingerprint,
|
||||||
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
|
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
|
||||||
prompt_token_ids=(
|
prompt_token_ids=(
|
||||||
final_res.prompt_token_ids if request.return_token_ids else None
|
final_res.prompt_token_ids if request.return_token_ids else None
|
||||||
|
|||||||
@@ -156,6 +156,18 @@ class BaseFrontendArgs:
|
|||||||
If set to True, only enable the Tokens In<>Out endpoint.
|
If set to True, only enable the Tokens In<>Out endpoint.
|
||||||
This is intended for use in a Disaggregated Everything setup.
|
This is intended for use in a Disaggregated Everything setup.
|
||||||
"""
|
"""
|
||||||
|
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
|
||||||
|
"""Controls the ``system_fingerprint`` field on responses.
|
||||||
|
|
||||||
|
- ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>``. Encodes
|
||||||
|
server version, non-trivial parallelism degrees (tp/pp/dp/ep), and an
|
||||||
|
8-char config hash.
|
||||||
|
- ``hash``: ``vllm-<version>-<hash8>``. Parallelism stripped.
|
||||||
|
- ``custom``: emits the literal string from ``--fingerprint-value``.
|
||||||
|
- ``none``: the field is omitted (serialized as ``null``).
|
||||||
|
"""
|
||||||
|
fingerprint_value: str | None = None
|
||||||
|
"""Literal fingerprint string used when ``--fingerprint-mode=custom``."""
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _customize_cli_kwargs(
|
def _customize_cli_kwargs(
|
||||||
|
|||||||
@@ -512,3 +512,6 @@ class CompletionStreamResponse(OpenAIBaseModel):
|
|||||||
model: str
|
model: str
|
||||||
choices: list[CompletionResponseStreamChoice]
|
choices: list[CompletionResponseStreamChoice]
|
||||||
usage: UsageInfo | None = Field(default=None)
|
usage: UsageInfo | None = Field(default=None)
|
||||||
|
# Set only on the final chunk of a stream to mirror non-streaming responses
|
||||||
|
# without the per-chunk serialization overhead.
|
||||||
|
system_fingerprint: str | None = None
|
||||||
|
|||||||
@@ -383,6 +383,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
|
|
||||||
chunk = CompletionStreamResponse(
|
chunk = CompletionStreamResponse(
|
||||||
id=request_id,
|
id=request_id,
|
||||||
|
object="text_completion",
|
||||||
created=created_time,
|
created=created_time,
|
||||||
model=model_name,
|
model=model_name,
|
||||||
choices=[
|
choices=[
|
||||||
@@ -401,6 +402,14 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
)
|
)
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
# Stamp on terminal chunk only when no trailing usage chunk
|
||||||
|
# will follow (that one is the true final message).
|
||||||
|
if (
|
||||||
|
not include_usage
|
||||||
|
and self.system_fingerprint is not None
|
||||||
|
and finish_reason is not None
|
||||||
|
):
|
||||||
|
chunk.system_fingerprint = self.system_fingerprint
|
||||||
if include_continuous_usage:
|
if include_continuous_usage:
|
||||||
prompt_tokens = num_prompt_tokens[prompt_idx]
|
prompt_tokens = num_prompt_tokens[prompt_idx]
|
||||||
completion_tokens = previous_num_tokens[i]
|
completion_tokens = previous_num_tokens[i]
|
||||||
@@ -410,7 +419,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
total_tokens=prompt_tokens + completion_tokens,
|
total_tokens=prompt_tokens + completion_tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
response_json = chunk.model_dump_json(exclude_unset=False)
|
response_json = chunk.model_dump_json(exclude_unset=True)
|
||||||
yield f"data: {response_json}\n\n"
|
yield f"data: {response_json}\n\n"
|
||||||
|
|
||||||
total_prompt_tokens = sum(num_prompt_tokens)
|
total_prompt_tokens = sum(num_prompt_tokens)
|
||||||
@@ -433,6 +442,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
model=model_name,
|
model=model_name,
|
||||||
choices=[],
|
choices=[],
|
||||||
usage=final_usage_info,
|
usage=final_usage_info,
|
||||||
|
system_fingerprint=self.system_fingerprint,
|
||||||
)
|
)
|
||||||
final_usage_data = final_usage_chunk.model_dump_json(
|
final_usage_data = final_usage_chunk.model_dump_json(
|
||||||
exclude_unset=False, exclude_none=True
|
exclude_unset=False, exclude_none=True
|
||||||
@@ -562,6 +572,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
|||||||
model=model_name,
|
model=model_name,
|
||||||
choices=choices,
|
choices=choices,
|
||||||
usage=usage,
|
usage=usage,
|
||||||
|
system_fingerprint=self.system_fingerprint,
|
||||||
kv_transfer_params=kv_transfer_params,
|
kv_transfer_params=kv_transfer_params,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -157,6 +157,19 @@ class OpenAIServing:
|
|||||||
self.renderer = engine_client.renderer
|
self.renderer = engine_client.renderer
|
||||||
self.input_processor = engine_client.input_processor
|
self.input_processor = engine_client.input_processor
|
||||||
|
|
||||||
|
# Computed once at startup (cached by ``vllm_config`` identity) and
|
||||||
|
# stamped on non-streaming responses. Streaming chunks deliberately
|
||||||
|
# omit it to avoid per-chunk overhead.
|
||||||
|
from vllm.entrypoints.openai.fingerprint import get_system_fingerprint
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.system_fingerprint: str | None = get_system_fingerprint(
|
||||||
|
engine_client.vllm_config
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# Never fail server startup over the fingerprint.
|
||||||
|
self.system_fingerprint = None
|
||||||
|
|
||||||
async def beam_search(
|
async def beam_search(
|
||||||
self,
|
self,
|
||||||
prompt: EngineInput,
|
prompt: EngineInput,
|
||||||
|
|||||||
@@ -0,0 +1,84 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Build the ``system_fingerprint`` string returned by the OpenAI-compatible
|
||||||
|
server.
|
||||||
|
|
||||||
|
Four modes, configured via ``--fingerprint-mode``:
|
||||||
|
|
||||||
|
* ``full`` (default): ``vllm-<version>[-<parallelism>]-<hash8>`` — encodes
|
||||||
|
server version, any non-trivial parallelism degree (tp/pp/dp/ep), and an
|
||||||
|
8-char prefix of ``vllm_config.compute_hash()`` (covers model identity,
|
||||||
|
quant config, speculative, attention backend, etc.).
|
||||||
|
* ``hash``: ``vllm-<version>-<hash8>`` — parallelism stripped.
|
||||||
|
* ``custom``: user-provided literal via ``--fingerprint-value``.
|
||||||
|
* ``none``: the field is omitted (serialized as ``null``).
|
||||||
|
|
||||||
|
``get_system_fingerprint`` is only called at serving-class init (a handful
|
||||||
|
of times per server); each subclass caches the returned string on
|
||||||
|
``self.system_fingerprint``, so per-request cost is one attribute read.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
FingerprintMode = Literal["full", "hash", "custom", "none"]
|
||||||
|
|
||||||
|
_DEFAULT_MODE: FingerprintMode = "full"
|
||||||
|
_CUSTOM_VALUE: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def set_default_fingerprint_mode(
|
||||||
|
mode: FingerprintMode,
|
||||||
|
custom_value: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Configure the fingerprint mode for subsequent ``get_system_fingerprint``
|
||||||
|
calls. Called once at server startup."""
|
||||||
|
global _DEFAULT_MODE, _CUSTOM_VALUE
|
||||||
|
_DEFAULT_MODE = mode
|
||||||
|
_CUSTOM_VALUE = custom_value
|
||||||
|
|
||||||
|
|
||||||
|
def get_system_fingerprint(vllm_config: Any) -> str | None:
|
||||||
|
"""Return the fingerprint for ``vllm_config`` using the mode configured by
|
||||||
|
``set_default_fingerprint_mode``."""
|
||||||
|
return build_system_fingerprint(vllm_config, _DEFAULT_MODE, _CUSTOM_VALUE)
|
||||||
|
|
||||||
|
|
||||||
|
def build_system_fingerprint(
|
||||||
|
vllm_config: Any,
|
||||||
|
mode: FingerprintMode = "full",
|
||||||
|
custom_value: str | None = None,
|
||||||
|
) -> str | None:
|
||||||
|
if mode == "none":
|
||||||
|
return None
|
||||||
|
if mode == "custom":
|
||||||
|
return custom_value
|
||||||
|
|
||||||
|
from vllm import __version__ as vllm_version
|
||||||
|
|
||||||
|
try:
|
||||||
|
hash8 = vllm_config.compute_hash()[:8]
|
||||||
|
except Exception:
|
||||||
|
hash8 = "nohash"
|
||||||
|
|
||||||
|
if mode == "hash":
|
||||||
|
return f"vllm-{vllm_version}-{hash8}"
|
||||||
|
|
||||||
|
# mode == "full"
|
||||||
|
parts: list[str] = [f"vllm-{vllm_version}"]
|
||||||
|
pc = getattr(vllm_config, "parallel_config", None)
|
||||||
|
if pc is not None:
|
||||||
|
tp = getattr(pc, "tensor_parallel_size", 1)
|
||||||
|
if tp > 1:
|
||||||
|
parts.append(f"tp{tp}")
|
||||||
|
pp = getattr(pc, "pipeline_parallel_size", 1)
|
||||||
|
if pp > 1:
|
||||||
|
parts.append(f"pp{pp}")
|
||||||
|
dp = getattr(pc, "data_parallel_size", 1)
|
||||||
|
if dp > 1:
|
||||||
|
parts.append(f"dp{dp}")
|
||||||
|
if getattr(pc, "enable_expert_parallel", False):
|
||||||
|
parts.append("ep")
|
||||||
|
parts.append(hash8)
|
||||||
|
return "-".join(parts)
|
||||||
@@ -61,9 +61,17 @@ async def init_generate_state(
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||||
|
from vllm.entrypoints.openai.fingerprint import set_default_fingerprint_mode
|
||||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||||
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
||||||
|
|
||||||
|
# Applied before any serving class is constructed so that each one picks
|
||||||
|
# up the chosen mode on its first cache miss.
|
||||||
|
set_default_fingerprint_mode(
|
||||||
|
getattr(args, "fingerprint_mode", "full"),
|
||||||
|
getattr(args, "fingerprint_value", None),
|
||||||
|
)
|
||||||
|
|
||||||
if args.tool_server == "demo":
|
if args.tool_server == "demo":
|
||||||
tool_server: ToolServer | None = DemoToolServer()
|
tool_server: ToolServer | None = DemoToolServer()
|
||||||
assert isinstance(tool_server, DemoToolServer)
|
assert isinstance(tool_server, DemoToolServer)
|
||||||
|
|||||||
Reference in New Issue
Block a user