[Feature] Universal speculative decoding for heterogeneous vocabularies (TLI) (#38174)
Signed-off-by: wan-danfeng <wandanfeng0802@gmail.com> Signed-off-by: Wonderful <wandanfeng0802@gmail.com> Co-authored-by: Wan_DF <wonderful199082@126.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
Wan_DF
Benjamin Chislett
Benjamin Chislett
parent
8357226f4f
commit
3af8789559
@@ -72,6 +72,7 @@ def parse_args():
|
||||
parser.add_argument("--max-num-seqs", type=int, default=None)
|
||||
parser.add_argument("--parallel-drafting", action="store_true")
|
||||
parser.add_argument("--allowed-local-media-path", type=str, default="")
|
||||
parser.add_argument("--use-heterogeneous-vocab", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -135,6 +136,7 @@ def main(args):
|
||||
"enforce_eager": args.enforce_eager,
|
||||
"max_model_len": args.max_model_len,
|
||||
"parallel_drafting": args.parallel_drafting,
|
||||
"use_heterogeneous_vocab": args.use_heterogeneous_vocab,
|
||||
}
|
||||
elif args.method == "mtp":
|
||||
speculative_config = {
|
||||
|
||||
Reference in New Issue
Block a user