[P/D] Prefill compute optimizations with bi-directional KV cache transfers between P and D nodes (#32553)

Signed-off-by: Sunita Nadampalli <nadampal@amazon.com>
This commit is contained in:
snadampal
2026-04-30 10:14:20 +00:00
committed by GitHub
parent efdc95674d
commit 3179e53135
4 changed files with 1570 additions and 9 deletions
@@ -217,6 +217,7 @@ async def send_request(
min_tokens: int | None = None,
max_tokens: int | None = None,
timeout_sec: int = 120,
conversation_id: str | None = None,
) -> ServerResponse:
payload = {
"model": model,
@@ -225,6 +226,9 @@ async def send_request(
"temperature": 0.0,
}
if conversation_id is not None:
payload["conversation_id"] = conversation_id
if stream:
payload["stream"] = True
payload["stream_options"] = {"include_usage": False}
@@ -419,6 +423,7 @@ async def send_turn(
min_tokens,
max_tokens,
req_args.timeout_sec,
conversation_id=conv_id,
)
if response.valid is False: