[EC Connector] Add EC Transfer Params (#42433)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -276,6 +276,7 @@ class RequestState:
|
||||
finish_reason: FinishReason | None,
|
||||
stop_reason: int | str | None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput | None:
|
||||
finished = finish_reason is not None
|
||||
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
|
||||
@@ -327,7 +328,11 @@ class RequestState:
|
||||
external_req_id = self.parent_req.external_req_id
|
||||
|
||||
return self._new_request_output(
|
||||
external_req_id, outputs, finished, kv_transfer_params
|
||||
external_req_id,
|
||||
outputs,
|
||||
finished,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
)
|
||||
|
||||
def _new_request_output(
|
||||
@@ -336,6 +341,7 @@ class RequestState:
|
||||
outputs: list[CompletionOutput] | list[PoolingOutput],
|
||||
finished: bool,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput:
|
||||
# If prompt embeds were used, put placeholder prompt token ids
|
||||
prompt_token_ids = self.prompt_token_ids
|
||||
@@ -369,6 +375,7 @@ class RequestState:
|
||||
outputs=cast(list[CompletionOutput], outputs),
|
||||
finished=finished,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
ec_transfer_params=ec_transfer_params,
|
||||
num_cached_tokens=self.num_cached_tokens,
|
||||
metrics=self.stats,
|
||||
)
|
||||
@@ -497,6 +504,7 @@ class OutputProcessor:
|
||||
finish_reason=FinishReason.ABORT,
|
||||
stop_reason=None,
|
||||
kv_transfer_params=None,
|
||||
ec_transfer_params=None,
|
||||
)
|
||||
):
|
||||
req_state.queue.put(request_output)
|
||||
@@ -620,6 +628,7 @@ class OutputProcessor:
|
||||
finish_reason = engine_core_output.finish_reason
|
||||
stop_reason = engine_core_output.stop_reason
|
||||
kv_transfer_params = engine_core_output.kv_transfer_params
|
||||
ec_transfer_params = engine_core_output.ec_transfer_params
|
||||
if engine_core_output.routed_experts is not None:
|
||||
req_state.routed_experts_chunks.append(
|
||||
engine_core_output.routed_experts
|
||||
@@ -654,6 +663,7 @@ class OutputProcessor:
|
||||
finish_reason,
|
||||
stop_reason,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
):
|
||||
if req_state.streaming_input:
|
||||
request_output.finished = False
|
||||
|
||||
Reference in New Issue
Block a user