[BugFix] Fix use of per-request seed with pipeline parallel (#6698)
This commit is contained in:
@@ -139,6 +139,9 @@ class ModelRunnerBase(ABC, Generic[T]):
|
||||
ModelRunnerInputBase subclass.
|
||||
"""
|
||||
|
||||
# Map of request_id -> generator used for seeded random sampling
|
||||
generators: Dict[str, torch.Generator] = {}
|
||||
|
||||
@abstractmethod
|
||||
def make_model_input_from_broadcasted_tensor_dict(
|
||||
self,
|
||||
@@ -176,3 +179,15 @@ class ModelRunnerBase(ABC, Generic[T]):
|
||||
Execute the model on the given input.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_generators(self, finished_request_ids: Optional[List[str]] = None):
|
||||
"""
|
||||
Return dict of per-request generators used for random sampling.
|
||||
"""
|
||||
|
||||
# Clean up generators from completed requests
|
||||
if finished_request_ids:
|
||||
for request_id in finished_request_ids:
|
||||
self.generators.pop(request_id, None)
|
||||
|
||||
return self.generators
|
||||
|
||||
Reference in New Issue
Block a user