feat: spec decode with draft models (#24322)

Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
Tomas Ruiz
2026-01-19 16:05:46 -05:00
committed by GitHub
parent 73f2a81c75
commit 4a5299c93f
21 changed files with 897 additions and 115 deletions
+10 -9
View File
@@ -2593,17 +2593,10 @@ class InstructCoderDataset(HuggingFaceDataset):
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
) -> list:
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
prompt = (
f"{item['input']}\n\n{item['instruction']} Just output "
"the code, do not include any explanation."
)
for i, prompt in enumerate(self.sample_prompts(n=num_requests)):
# apply template
if not skip_chat_template:
prompt = tokenizer.apply_chat_template(
@@ -2626,6 +2619,14 @@ class InstructCoderDataset(HuggingFaceDataset):
)
return sampled_requests
def sample_prompts(self, n: int) -> Iterator[str]:
for item in self.data.take(n):
prompt = (
f"{item['input']}\n\n{item['instruction']} Just output "
"the code, do not include any explanation."
)
yield prompt
# -----------------------------------------------------------------------------
# MT-Bench Dataset Implementation