Runtime Draft Weight Update for Speculative Decoding (#46725)
Signed-off-by: vx120 <893600387@qq.com> Signed-off-by: vx120 <57470515+vx120@users.noreply.github.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: crp0128 <191679376@qq.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
crp0128
aoshen02
Claude Opus 4.8
parent
481e481be7
commit
fc1c548093
@@ -877,6 +877,10 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
"""Start a new weight update."""
|
||||
self.llm_engine.collective_rpc("start_weight_update")
|
||||
|
||||
def start_draft_weight_update(self) -> None:
|
||||
"""Start a new weight update targeting the speculative draft model."""
|
||||
self.llm_engine.collective_rpc("start_draft_weight_update")
|
||||
|
||||
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
|
||||
"""
|
||||
Update the weights of the model.
|
||||
|
||||
@@ -133,6 +133,12 @@ async def start_weight_update(raw_request: Request):
|
||||
return JSONResponse(content={"message": "Weight update started"})
|
||||
|
||||
|
||||
@router.post("/start_draft_weight_update")
|
||||
async def start_draft_weight_update(raw_request: Request):
|
||||
await engine_client(raw_request).start_draft_weight_update()
|
||||
return JSONResponse(content={"message": "Draft weight update started"})
|
||||
|
||||
|
||||
@router.post("/update_weights")
|
||||
async def update_weights(raw_request: Request):
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user