diff --git a/vllm/entrypoints/anthropic/protocol.py b/vllm/entrypoints/anthropic/protocol.py index ab3ca66e2c..3445f70910 100644 --- a/vllm/entrypoints/anthropic/protocol.py +++ b/vllm/entrypoints/anthropic/protocol.py @@ -5,7 +5,7 @@ import time from typing import Any, Literal -from pydantic import BaseModel, field_validator, model_validator +from pydantic import BaseModel, Field, field_validator, model_validator class AnthropicError(BaseModel): @@ -112,6 +112,12 @@ class AnthropicMessagesRequest(BaseModel): top_k: int | None = None top_p: float | None = None + # vLLM-specific fields that are not in Anthropic spec + kv_transfer_params: dict[str, Any] | None = Field( + default=None, + description="KVTransfer parameters used for disaggregated serving.", + ) + @field_validator("model") @classmethod def validate_model(cls, v): @@ -181,6 +187,11 @@ class AnthropicMessagesResponse(BaseModel): stop_sequence: str | None = None usage: AnthropicUsage | None = None + # vLLM-specific fields that are not in Anthropic spec + kv_transfer_params: dict[str, Any] | None = Field( + default=None, description="KVTransfer parameters." + ) + def model_post_init(self, __context): if not self.id: self.id = f"msg_{int(time.time() * 1000)}" diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 38601b6bff..4b495168c1 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -331,6 +331,7 @@ class AnthropicServingMessages(OpenAIServingChat): temperature=anthropic_request.temperature, top_p=anthropic_request.top_p, top_k=anthropic_request.top_k, + kv_transfer_params=anthropic_request.kv_transfer_params, ) @classmethod @@ -441,6 +442,7 @@ class AnthropicServingMessages(OpenAIServingChat): input_tokens=generator.usage.prompt_tokens, output_tokens=generator.usage.completion_tokens, ), + kv_transfer_params=generator.kv_transfer_params, ) choice = generator.choices[0] if choice.finish_reason == "stop":