Runtime Draft Weight Update for Speculative Decoding (#46725)

Signed-off-by: vx120 <893600387@qq.com>
Signed-off-by: vx120 <57470515+vx120@users.noreply.github.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: crp0128 <191679376@qq.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
vx120
2026-07-11 22:51:53 -07:00
committed by GitHub
co-authored by crp0128 aoshen02 Claude Opus 4.8
parent 481e481be7
commit fc1c548093
12 changed files with 129 additions and 1 deletions
+4
View File
@@ -49,6 +49,10 @@ update_request = WeightTransferUpdateRequest(
)
```
At the LLM/API layer, call `start_draft_weight_update()` instead of
`start_weight_update()` to target the speculative draft model;
`update_weights` / `finish_weight_update` are unchanged.
### WeightTransferUpdateInfo
The base `WeightTransferUpdateInfo` is a marker class for backend-specific update info:
@@ -145,6 +145,7 @@ def test_openapi_stateless(case: schemathesis.Case):
if case.operation.path in (
"/init_weight_transfer_engine",
"/start_weight_update",
"/start_draft_weight_update",
"/update_weights",
"/finish_weight_update",
):
@@ -19,6 +19,7 @@ class _RecordingEngine:
self.raise_on_update = raise_on_update
self.started = False
self.finished = False
self.reset_count = 0
self.update_calls: list[dict] = []
def start_weight_update(self) -> None:
@@ -32,6 +33,9 @@ class _RecordingEngine:
def finish_weight_update(self) -> None:
self.finished = True
def reset_weight_update_target(self) -> None:
self.reset_count += 1
def _make_worker(engine: _RecordingEngine | None) -> Worker:
worker = object.__new__(Worker)
@@ -54,6 +58,7 @@ def test_start_update_finish_delegates_to_engine():
Worker.finish_weight_update(worker)
assert engine.finished is True
assert engine.reset_count == 1
assert worker._weight_update_active is False
@@ -85,6 +90,7 @@ def test_update_resets_active_on_error():
Worker.update_weights(worker, {"names": ["w"]})
# A failed update ends the session so the next start is clean.
assert engine.reset_count == 1
assert worker._weight_update_active is False
+18
View File
@@ -73,6 +73,8 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
init_info_cls: type[TInitInfo]
update_info_cls: type[TUpdateInfo]
supports_draft_weight_update: bool = True
def __init__(
self,
config: WeightTransferConfig,
@@ -95,6 +97,22 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
self.model_config = vllm_config.model_config
self.device = device
self.model = model
self._default_model_config = self.model_config
self._default_model = model
def set_weight_update_target(
self,
model: torch.nn.Module,
model_config: Any,
) -> None:
"""Set the model that will receive the active weight update."""
self.model = model
self.model_config = model_config
def reset_weight_update_target(self) -> None:
"""Restore weight updates to the engine's default target model."""
self.model = self._default_model
self.model_config = self._default_model_config
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
"""
@@ -99,6 +99,7 @@ class SparseNCCLWeightTransferEngine(
init_info_cls = NCCLWeightTransferInitInfo
update_info_cls = SparseNCCLWeightTransferUpdateInfo
supports_draft_weight_update = False
def __init__(
self,
+4
View File
@@ -248,6 +248,10 @@ class EngineClient(ABC):
"""Start a new weight update."""
raise NotImplementedError
async def start_draft_weight_update(self) -> None:
"""Start a new weight update targeting the speculative draft model."""
raise NotImplementedError
async def update_weights(self, request: WeightTransferUpdateRequest) -> None:
"""Batched weight update for RL training."""
raise NotImplementedError
+4
View File
@@ -877,6 +877,10 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
"""Start a new weight update."""
self.llm_engine.collective_rpc("start_weight_update")
def start_draft_weight_update(self) -> None:
"""Start a new weight update targeting the speculative draft model."""
self.llm_engine.collective_rpc("start_draft_weight_update")
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
"""
Update the weights of the model.
@@ -133,6 +133,12 @@ async def start_weight_update(raw_request: Request):
return JSONResponse(content={"message": "Weight update started"})
@router.post("/start_draft_weight_update")
async def start_draft_weight_update(raw_request: Request):
await engine_client(raw_request).start_draft_weight_update()
return JSONResponse(content={"message": "Draft weight update started"})
@router.post("/update_weights")
async def update_weights(raw_request: Request):
try:
+4
View File
@@ -1084,6 +1084,10 @@ class AsyncLLM(EngineClient):
"""Start a new weight update."""
await self.collective_rpc("start_weight_update")
async def start_draft_weight_update(self) -> None:
"""Start a new weight update targeting the speculative draft model."""
await self.collective_rpc("start_draft_weight_update")
async def update_weights(self, request: WeightTransferUpdateRequest) -> None:
"""
Batched weight update for RL training.
+6
View File
@@ -371,6 +371,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def get_model(self) -> nn.Module:
return self.model
def get_draft_model(self) -> nn.Module | None:
speculator = self.speculator
if not isinstance(speculator, DraftModelSpeculator):
return None
return speculator.model
def reload_weights(self, *args, **kwargs) -> None:
# TODO(Wentao): Use full version instead of import when fully migrated to v2
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1
+11
View File
@@ -3282,6 +3282,17 @@ class GPUModelRunner(
return self.model.unwrap()
return self.model
def get_draft_model(self) -> nn.Module | None:
drafter = getattr(self, "drafter", None)
if drafter is None:
return None
model = getattr(drafter, "model", None)
if isinstance(
model, (CUDAGraphWrapper, UBatchWrapper, BreakableCUDAGraphWrapper)
):
return cast(nn.Module, model.unwrap())
return cast(nn.Module | None, model)
def get_supported_generation_tasks(self) -> list[GenerationTask]:
model = self.get_model()
supported_tasks = list[GenerationTask]()
+64 -1
View File
@@ -154,6 +154,7 @@ class Worker(WorkerBase):
# Buffers saved before sleep
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
self._sleep_rebuild_draft_metadata_buffers = False
# Weight transfer engine is created in `load_model` once the model
# is available, since the engine needs a reference to the model.
@@ -198,6 +199,11 @@ class Worker(WorkerBase):
self._sleep_saved_buffers = {
name: buffer.cpu().clone() for name, buffer in model.named_buffers()
}
draft = self.get_draft_model()
inner = getattr(draft, "model", None) if draft is not None else None
self._sleep_rebuild_draft_metadata_buffers = inner is not None and hasattr(
inner, "_build_fused_kv_buffers"
)
self._get_sleep_mode_backend().suspend(level)
@@ -229,6 +235,14 @@ class Worker(WorkerBase):
buffer.data.copy_(self._sleep_saved_buffers[name].data)
self._sleep_saved_buffers = {}
if self._sleep_rebuild_draft_metadata_buffers:
draft = self.get_draft_model()
if draft is not None:
inner = getattr(draft, "model", None)
if inner is not None and hasattr(inner, "_build_fused_kv_buffers"):
inner._build_fused_kv_buffers()
self._sleep_rebuild_draft_metadata_buffers = False
if tags is None or "kv_cache" in tags:
self.model_runner.post_kv_cache_wake_up()
@@ -915,6 +929,29 @@ class Worker(WorkerBase):
def get_model(self) -> nn.Module:
return self.model_runner.get_model()
def get_draft_model(self) -> nn.Module | None:
return self.model_runner.get_draft_model()
def _set_draft_weight_update_target(self) -> None:
assert self.weight_transfer_engine is not None
draft_model = self.get_draft_model()
if draft_model is None:
raise RuntimeError(
"Draft model weight update requested, but no draft model is configured."
)
speculative_config = self.speculative_config
if speculative_config is None or speculative_config.draft_model_config is None:
raise RuntimeError(
"Draft model weight update requested, but no draft model "
"config is configured."
)
self.weight_transfer_engine.set_weight_update_target(
draft_model, speculative_config.draft_model_config
)
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
return self.model_runner.get_supported_tasks()
@@ -1179,16 +1216,38 @@ class Worker(WorkerBase):
the configured weight transfer engine. The worker only tracks that a
session is active.
"""
self._start_weight_update()
def start_draft_weight_update(self) -> None:
"""
Like start_weight_update, but retargets the engine at the speculative
draft model for this session.
"""
self._start_weight_update(is_draft=True)
def _start_weight_update(self, is_draft: bool = False) -> None:
self._check_weight_transfer_engine()
assert self.weight_transfer_engine is not None
if is_draft and not self.weight_transfer_engine.supports_draft_weight_update:
raise RuntimeError(
f"{type(self.weight_transfer_engine).__name__} does not support "
"draft model weight updates."
)
if self._weight_update_active:
raise RuntimeError(
"start_weight_update called while a weight update is already "
"active. Call finish_weight_update first."
)
self.weight_transfer_engine.start_weight_update()
try:
if is_draft:
self._set_draft_weight_update_target()
self.weight_transfer_engine.start_weight_update()
except BaseException:
self.weight_transfer_engine.reset_weight_update_target()
raise
self._weight_update_active = True
def update_weights(self, update_info: dict) -> None:
@@ -1197,6 +1256,8 @@ class Worker(WorkerBase):
start_weight_update must be called before update_weights and
finish_weight_update must be called after all chunks have been sent.
Every chunk loads into whichever model the session's start_weight_update
/ start_draft_weight_update call selected.
Args:
update_info: Dictionary containing backend-specific update info
@@ -1213,6 +1274,7 @@ class Worker(WorkerBase):
self.weight_transfer_engine.update_weights(update_info)
except BaseException:
self._weight_update_active = False
self.weight_transfer_engine.reset_weight_update_target()
raise
def finish_weight_update(self) -> None:
@@ -1226,6 +1288,7 @@ class Worker(WorkerBase):
)
self.weight_transfer_engine.finish_weight_update()
self.weight_transfer_engine.reset_weight_update_target()
self._weight_update_active = False
def shutdown(self) -> None: