Migrate logits computation and gather to model_runner (#3233)

This commit is contained in:
Roy
2024-03-20 23:25:01 +00:00
committed by GitHub
parent 6e435de766
commit f1c0fc3919
35 changed files with 576 additions and 305 deletions
+8 -1
View File
@@ -613,9 +613,16 @@ class ModelRunner:
input_metadata=input_metadata,
)
# Compute the logits.
logits = self.model.compute_logits(hidden_states, sampling_metadata)
# Only perform sampling in the driver worker.
if not sampling_metadata.perform_sampling:
return None
# Sample the next token.
output = self.model.sample(
hidden_states=hidden_states,
logits=logits,
sampling_metadata=sampling_metadata,
)
return output