[V1] [P/D] Refactor KV Connector Path (#21980)
Signed-off-by: David Ben-David <davidb@pliops.com> Co-authored-by: David Ben-David <davidb@pliops.com>
This commit is contained in:
co-authored by
David Ben-David
parent
24d1dffbeb
commit
aefeea0fde
+8
-5
@@ -71,6 +71,13 @@ class SamplerOutput:
|
||||
logprobs_tensors: Optional[LogprobsTensors]
|
||||
|
||||
|
||||
@dataclass
|
||||
class KVConnectorOutput:
|
||||
# [req_ids]
|
||||
finished_sending: Optional[set[str]] = None
|
||||
finished_recving: Optional[set[str]] = None
|
||||
|
||||
|
||||
# ModelRunnerOutput is serialized and sent to the scheduler process.
|
||||
# This is expensive for torch.Tensor so prefer to use list instead.
|
||||
@dataclass
|
||||
@@ -104,9 +111,7 @@ class ModelRunnerOutput:
|
||||
# [num_reqs, hidden_size]
|
||||
pooler_output: list[Optional[torch.Tensor]]
|
||||
|
||||
# [req_ids]
|
||||
finished_sending: Optional[set[str]] = None
|
||||
finished_recving: Optional[set[str]] = None
|
||||
kv_connector_output: Optional[KVConnectorOutput] = None
|
||||
|
||||
# req_id -> num_nans_in_logits
|
||||
num_nans_in_logits: Optional[dict[str, int]] = None
|
||||
@@ -119,6 +124,4 @@ EMPTY_MODEL_RUNNER_OUTPUT = ModelRunnerOutput(req_ids=[],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
finished_sending=None,
|
||||
finished_recving=None,
|
||||
num_nans_in_logits=None)
|
||||
|
||||
Reference in New Issue
Block a user