[Core] Pipeline Parallel Support (#4412)
Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
This commit is contained in:
@@ -770,6 +770,34 @@ class EmbeddingSequenceGroupOutput(SequenceGroupOutput):
|
||||
return self.embeddings == other.embeddings
|
||||
|
||||
|
||||
@dataclass
|
||||
class IntermediateTensors:
|
||||
"""For all pipeline stages except the last, we need to return the hidden
|
||||
states and residuals to be sent to the next stage. This data structure
|
||||
contains the hidden states and residuals for a request.
|
||||
"""
|
||||
|
||||
tensors: Dict[str, torch.Tensor]
|
||||
|
||||
def __getitem__(self, key: Union[str, slice]):
|
||||
if isinstance(key, str):
|
||||
return self.tensors[key]
|
||||
elif isinstance(key, slice):
|
||||
return self.__class__({k: v[key] for k, v in self.tensors.items()})
|
||||
|
||||
def __setitem__(self, key: str, value):
|
||||
self.tensors[key] = value
|
||||
|
||||
def __len__(self):
|
||||
return len(self.tensors)
|
||||
|
||||
def __eq__(self, other: object):
|
||||
return isinstance(other, self.__class__) and self
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"IntermediateTensors(tensors={self.tensors})"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SamplerOutput:
|
||||
"""For each sequence group, we generate a list of SequenceOutput object,
|
||||
@@ -896,6 +924,8 @@ class ExecuteModelRequest:
|
||||
blocks_to_swap_out: List[Tuple[int, int]] = field(default_factory=list)
|
||||
# Blocks to copy. Source to dest block.
|
||||
blocks_to_copy: List[Tuple[int, int]] = field(default_factory=list)
|
||||
# Virtual engine ID for pipeline parallel.
|
||||
virtual_engine: int = 0
|
||||
# The number of slots for lookahead decoding.
|
||||
num_lookahead_slots: int = 0
|
||||
# The number of requests in the running queue.
|
||||
@@ -914,6 +944,7 @@ class ExecuteModelRequest:
|
||||
blocks_to_swap_in=self.blocks_to_swap_in.copy(),
|
||||
blocks_to_swap_out=self.blocks_to_swap_out.copy(),
|
||||
blocks_to_copy=self.blocks_to_copy.copy(),
|
||||
virtual_engine=self.virtual_engine,
|
||||
num_lookahead_slots=self.num_lookahead_slots,
|
||||
running_queue_size=self.running_queue_size,
|
||||
previous_hidden_states=self.previous_hidden_states,
|
||||
|
||||
Reference in New Issue
Block a user