feat: spec decode with draft models (#24322)

Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
Tomas Ruiz
2026-01-19 16:05:46 -05:00
committed by GitHub
parent 73f2a81c75
commit 4a5299c93f
21 changed files with 897 additions and 115 deletions
+10 -9
View File
@@ -2593,17 +2593,10 @@ class InstructCoderDataset(HuggingFaceDataset):
request_id_prefix: str = "",
no_oversample: bool = False,
**kwargs,
) -> list:
) -> list[SampleRequest]:
output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN
sampled_requests = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
prompt = (
f"{item['input']}\n\n{item['instruction']} Just output "
"the code, do not include any explanation."
)
for i, prompt in enumerate(self.sample_prompts(n=num_requests)):
# apply template
if not skip_chat_template:
prompt = tokenizer.apply_chat_template(
@@ -2626,6 +2619,14 @@ class InstructCoderDataset(HuggingFaceDataset):
)
return sampled_requests
def sample_prompts(self, n: int) -> Iterator[str]:
for item in self.data.take(n):
prompt = (
f"{item['input']}\n\n{item['instruction']} Just output "
"the code, do not include any explanation."
)
yield prompt
# -----------------------------------------------------------------------------
# MT-Bench Dataset Implementation
+6
View File
@@ -8,8 +8,12 @@ import time
import aiohttp
from tqdm.asyncio import tqdm
from vllm.logger import init_logger
from .endpoint_request_func import RequestFunc, RequestFuncInput, RequestFuncOutput
logger = init_logger(__name__)
async def wait_for_endpoint(
request_func: RequestFunc,
@@ -61,6 +65,8 @@ async def wait_for_endpoint(
if output.success:
pbar.close()
return output
else:
logger.warning("Endpoint is not ready. Error='%s'", output.error)
except aiohttp.ClientConnectorError:
pass