[EC Connector] Add EC Transfer Params (#42433)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -100,6 +100,8 @@ class Request:
|
||||
|
||||
# P/D: Connector-specific KV transfer parameters.
|
||||
self.kv_transfer_params: dict[str, Any] | None = None
|
||||
# E/P/D: Connector-specific encoder-cache transfer parameters.
|
||||
self.ec_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
if pooling_params is not None:
|
||||
# Pooling models.
|
||||
@@ -115,6 +117,9 @@ class Request:
|
||||
self.kv_transfer_params = sampling_params.extra_args.get(
|
||||
"kv_transfer_params"
|
||||
)
|
||||
self.ec_transfer_params = sampling_params.extra_args.get(
|
||||
"ec_transfer_params"
|
||||
)
|
||||
self.kv_cache_report_mode = sampling_params.extra_args.get(
|
||||
"kv_cache_report_mode", "incremental"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user