Revert "[V1] Exception Handling when Loading KV Cache from Remote Store" (#21778)
Signed-off-by: KuntaiDu <kuntai@uchicago.edu>
This commit is contained in:
@@ -139,27 +139,13 @@ class KVOutputAggregator:
|
||||
finished_set.add(req_id)
|
||||
del remaining_count_dict[req_id]
|
||||
|
||||
def update_finished_load_dict(worker_finished_loading_dict: dict[str,
|
||||
int],
|
||||
finished_loading_dict: dict[str, int]):
|
||||
for req_id, num_actual_load_tokens in (worker_finished_loading_dict
|
||||
or {}).items():
|
||||
if req_id in finished_loading_dict:
|
||||
finished_loading_dict[req_id] = min(
|
||||
finished_loading_dict[req_id], num_actual_load_tokens)
|
||||
else:
|
||||
finished_loading_dict[req_id] = num_actual_load_tokens
|
||||
|
||||
finished_sending = set[str]()
|
||||
finished_recving = set[str]()
|
||||
finished_loading_dict: dict[str, int] = {}
|
||||
for output in outputs:
|
||||
update_finished_set(output.finished_sending,
|
||||
self._send_remaining_count, finished_sending)
|
||||
update_finished_set(output.finished_recving,
|
||||
self._recv_remaining_count, finished_recving)
|
||||
update_finished_load_dict(output.finished_loading_dict,
|
||||
finished_loading_dict)
|
||||
|
||||
# select output of the worker specified by output_rank
|
||||
output = outputs[output_rank]
|
||||
@@ -171,7 +157,7 @@ class KVOutputAggregator:
|
||||
# send/recv
|
||||
output.finished_sending = finished_sending if finished_sending else None
|
||||
output.finished_recving = finished_recving if finished_recving else None
|
||||
output.finished_loading_dict = finished_loading_dict or None
|
||||
|
||||
return output
|
||||
|
||||
def async_aggregate(self,
|
||||
|
||||
@@ -28,9 +28,6 @@ The class provides the following primitives:
|
||||
|
||||
get_finished() - called with ids of finished requests, returns
|
||||
ids of requests that have completed async sending/recving.
|
||||
get_finished_loading() - called with scheduler outputs, returns
|
||||
a dictionary that the keys are request IDs and the values are
|
||||
the actual number of tokens loaded from the remote KV cache
|
||||
"""
|
||||
|
||||
import enum
|
||||
@@ -222,23 +219,6 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return None, None
|
||||
|
||||
def get_finished_loading(
|
||||
self, scheduler_output: "SchedulerOutput") -> dict[str, int]:
|
||||
"""
|
||||
Retrieves the actual number of tokens loaded for requests that have
|
||||
completed the asynchronous loading process from the remote KV cache.
|
||||
|
||||
This function is used by the scheduler process (via the Executors)
|
||||
to track the progress of requests and determine which requests have
|
||||
successfully finished loading their KV cache data.
|
||||
|
||||
Returns:
|
||||
A dictionary where the keys are request IDs and the values are the
|
||||
corresponding number of tokens that have been successfully loaded
|
||||
for each request.
|
||||
"""
|
||||
return {}
|
||||
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
# ==============================
|
||||
|
||||
Reference in New Issue
Block a user