[Core] [Bugfix] Add Input Embeddings (#15428)
Signed-off-by: Andrew Sansom <andrew@protopia.ai> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: 临景 <linjing.yx@alibaba-inc.com> Co-authored-by: Bryce1010 <bryceyx@gmail.com> Co-authored-by: Nan2018 <nan@protopia.ai> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
临景
Bryce1010
Nan2018
Cyrus Leung
DarkLight1337
parent
9e2de9b9e9
commit
cc2a77d7f1
@@ -110,6 +110,11 @@ class SamplerOutput(
|
||||
# 'broadcasted' to all other PP ranks for next step.
|
||||
sampled_token_ids_cpu: Optional[torch.Tensor] = None
|
||||
|
||||
# On-device tensor containing the sampled token embeddings (embeddings
|
||||
# corresponding to the sampled token ids). Used when prompt embeddings are
|
||||
# specified in lieu of prompt token ids or text.
|
||||
sampled_token_embeds: Optional[torch.Tensor] = None
|
||||
|
||||
# Spec decode metrics populated by workers.
|
||||
spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
|
||||
|
||||
@@ -183,7 +188,7 @@ class Sampler(nn.Module):
|
||||
|
||||
# Whether or not the SamplerOutput should have on-device tensors
|
||||
# containing the sampled token ids and probabilities. This is used by
|
||||
# speculative decoding.
|
||||
# speculative decoding and when prompt embeddings are specified.
|
||||
self.include_gpu_probs_tensor = False
|
||||
self.should_modify_greedy_probs_inplace = False
|
||||
|
||||
|
||||
Reference in New Issue
Block a user