[Feature] Universal speculative decoding for heterogeneous vocabularies (TLI) (#38174)

Signed-off-by: wan-danfeng <wandanfeng0802@gmail.com>
Signed-off-by: Wonderful <wandanfeng0802@gmail.com>
Co-authored-by: Wan_DF <wonderful199082@126.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Wonderful
2026-07-02 01:34:20 -07:00
committed by GitHub
co-authored by Wan_DF Benjamin Chislett Benjamin Chislett
parent 8357226f4f
commit 3af8789559
8 changed files with 364 additions and 5 deletions
@@ -72,6 +72,7 @@ def parse_args():
parser.add_argument("--max-num-seqs", type=int, default=None)
parser.add_argument("--parallel-drafting", action="store_true")
parser.add_argument("--allowed-local-media-path", type=str, default="")
parser.add_argument("--use-heterogeneous-vocab", action="store_true")
return parser.parse_args()
@@ -135,6 +136,7 @@ def main(args):
"enforce_eager": args.enforce_eager,
"max_model_len": args.max_model_len,
"parallel_drafting": args.parallel_drafting,
"use_heterogeneous_vocab": args.use_heterogeneous_vocab,
}
elif args.method == "mtp":
speculative_config = {