[EC Connector] Add EC Transfer Params (#42433)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
omerpaz95
2026-07-12 14:35:33 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 370b678a02
commit 8df14cfc8c
55 changed files with 416 additions and 12 deletions
+4
View File
@@ -104,6 +104,7 @@ class RequestOutput:
None if decoder-only.
num_cached_tokens: The number of tokens with prefix cache hit.
kv_transfer_params: The params for remote K/V transfer.
ec_transfer_params: The params for remote encoder-cache transfer.
"""
def __init__(
@@ -121,6 +122,7 @@ class RequestOutput:
num_cached_tokens: int | None = None,
*,
kv_transfer_params: dict[str, Any] | None = None,
ec_transfer_params: dict[str, Any] | None = None,
# Forward compatibility, code that uses args added in new release can
# still run with older versions of vLLM without breaking.
**kwargs: Any,
@@ -141,12 +143,14 @@ class RequestOutput:
self.encoder_prompt_token_ids = encoder_prompt_token_ids
self.num_cached_tokens = num_cached_tokens
self.kv_transfer_params = kv_transfer_params
self.ec_transfer_params = ec_transfer_params
def add(self, next_output: "RequestOutput", aggregate: bool) -> None:
"""Merge subsequent RequestOutput into this one"""
self.finished |= next_output.finished
self.kv_transfer_params = next_output.kv_transfer_params
self.ec_transfer_params = next_output.ec_transfer_params
for next_completion in next_output.outputs:
for i, completion in enumerate(self.outputs):