[P/D] AnthropicMessages add kv_transfer_params for PD disaggregation (#37535)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-03-19 16:41:20 +00:00
committed by GitHub
parent 2890aecce5
commit 2f9f946b22
2 changed files with 14 additions and 1 deletions
+12 -1
View File
@@ -5,7 +5,7 @@
import time
from typing import Any, Literal
from pydantic import BaseModel, field_validator, model_validator
from pydantic import BaseModel, Field, field_validator, model_validator
class AnthropicError(BaseModel):
@@ -112,6 +112,12 @@ class AnthropicMessagesRequest(BaseModel):
top_k: int | None = None
top_p: float | None = None
# vLLM-specific fields that are not in Anthropic spec
kv_transfer_params: dict[str, Any] | None = Field(
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
@field_validator("model")
@classmethod
def validate_model(cls, v):
@@ -181,6 +187,11 @@ class AnthropicMessagesResponse(BaseModel):
stop_sequence: str | None = None
usage: AnthropicUsage | None = None
# vLLM-specific fields that are not in Anthropic spec
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
def model_post_init(self, __context):
if not self.id:
self.id = f"msg_{int(time.time() * 1000)}"
+2
View File
@@ -331,6 +331,7 @@ class AnthropicServingMessages(OpenAIServingChat):
temperature=anthropic_request.temperature,
top_p=anthropic_request.top_p,
top_k=anthropic_request.top_k,
kv_transfer_params=anthropic_request.kv_transfer_params,
)
@classmethod
@@ -441,6 +442,7 @@ class AnthropicServingMessages(OpenAIServingChat):
input_tokens=generator.usage.prompt_tokens,
output_tokens=generator.usage.completion_tokens,
),
kv_transfer_params=generator.kv_transfer_params,
)
choice = generator.choices[0]
if choice.finish_reason == "stop":