[P/D] AnthropicMessages add kv_transfer_params for PD disaggregation (#37535)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -5,7 +5,7 @@
|
||||
import time
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, field_validator, model_validator
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
|
||||
class AnthropicError(BaseModel):
|
||||
@@ -112,6 +112,12 @@ class AnthropicMessagesRequest(BaseModel):
|
||||
top_k: int | None = None
|
||||
top_p: float | None = None
|
||||
|
||||
# vLLM-specific fields that are not in Anthropic spec
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def validate_model(cls, v):
|
||||
@@ -181,6 +187,11 @@ class AnthropicMessagesResponse(BaseModel):
|
||||
stop_sequence: str | None = None
|
||||
usage: AnthropicUsage | None = None
|
||||
|
||||
# vLLM-specific fields that are not in Anthropic spec
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
|
||||
def model_post_init(self, __context):
|
||||
if not self.id:
|
||||
self.id = f"msg_{int(time.time() * 1000)}"
|
||||
|
||||
@@ -331,6 +331,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
temperature=anthropic_request.temperature,
|
||||
top_p=anthropic_request.top_p,
|
||||
top_k=anthropic_request.top_k,
|
||||
kv_transfer_params=anthropic_request.kv_transfer_params,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -441,6 +442,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
input_tokens=generator.usage.prompt_tokens,
|
||||
output_tokens=generator.usage.completion_tokens,
|
||||
),
|
||||
kv_transfer_params=generator.kv_transfer_params,
|
||||
)
|
||||
choice = generator.choices[0]
|
||||
if choice.finish_reason == "stop":
|
||||
|
||||
Reference in New Issue
Block a user