[EC Connector] Add EC Transfer Params (#42433)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
omerpaz95
2026-07-12 14:35:33 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 370b678a02
commit 8df14cfc8c
55 changed files with 416 additions and 12 deletions
+11 -1
View File
@@ -276,6 +276,7 @@ class RequestState:
finish_reason: FinishReason | None,
stop_reason: int | str | None,
kv_transfer_params: dict[str, Any] | None = None,
ec_transfer_params: dict[str, Any] | None = None,
) -> RequestOutput | PoolingRequestOutput | None:
finished = finish_reason is not None
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
@@ -327,7 +328,11 @@ class RequestState:
external_req_id = self.parent_req.external_req_id
return self._new_request_output(
external_req_id, outputs, finished, kv_transfer_params
external_req_id,
outputs,
finished,
kv_transfer_params,
ec_transfer_params,
)
def _new_request_output(
@@ -336,6 +341,7 @@ class RequestState:
outputs: list[CompletionOutput] | list[PoolingOutput],
finished: bool,
kv_transfer_params: dict[str, Any] | None = None,
ec_transfer_params: dict[str, Any] | None = None,
) -> RequestOutput | PoolingRequestOutput:
# If prompt embeds were used, put placeholder prompt token ids
prompt_token_ids = self.prompt_token_ids
@@ -369,6 +375,7 @@ class RequestState:
outputs=cast(list[CompletionOutput], outputs),
finished=finished,
kv_transfer_params=kv_transfer_params,
ec_transfer_params=ec_transfer_params,
num_cached_tokens=self.num_cached_tokens,
metrics=self.stats,
)
@@ -497,6 +504,7 @@ class OutputProcessor:
finish_reason=FinishReason.ABORT,
stop_reason=None,
kv_transfer_params=None,
ec_transfer_params=None,
)
):
req_state.queue.put(request_output)
@@ -620,6 +628,7 @@ class OutputProcessor:
finish_reason = engine_core_output.finish_reason
stop_reason = engine_core_output.stop_reason
kv_transfer_params = engine_core_output.kv_transfer_params
ec_transfer_params = engine_core_output.ec_transfer_params
if engine_core_output.routed_experts is not None:
req_state.routed_experts_chunks.append(
engine_core_output.routed_experts
@@ -654,6 +663,7 @@ class OutputProcessor:
finish_reason,
stop_reason,
kv_transfer_params,
ec_transfer_params,
):
if req_state.streaming_input:
request_output.finished = False