Runtime Draft Weight Update for Speculative Decoding (#46725)
Signed-off-by: vx120 <893600387@qq.com> Signed-off-by: vx120 <57470515+vx120@users.noreply.github.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: crp0128 <191679376@qq.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
crp0128
aoshen02
Claude Opus 4.8
parent
481e481be7
commit
fc1c548093
@@ -49,6 +49,10 @@ update_request = WeightTransferUpdateRequest(
|
||||
)
|
||||
```
|
||||
|
||||
At the LLM/API layer, call `start_draft_weight_update()` instead of
|
||||
`start_weight_update()` to target the speculative draft model;
|
||||
`update_weights` / `finish_weight_update` are unchanged.
|
||||
|
||||
### WeightTransferUpdateInfo
|
||||
|
||||
The base `WeightTransferUpdateInfo` is a marker class for backend-specific update info:
|
||||
|
||||
@@ -145,6 +145,7 @@ def test_openapi_stateless(case: schemathesis.Case):
|
||||
if case.operation.path in (
|
||||
"/init_weight_transfer_engine",
|
||||
"/start_weight_update",
|
||||
"/start_draft_weight_update",
|
||||
"/update_weights",
|
||||
"/finish_weight_update",
|
||||
):
|
||||
|
||||
@@ -19,6 +19,7 @@ class _RecordingEngine:
|
||||
self.raise_on_update = raise_on_update
|
||||
self.started = False
|
||||
self.finished = False
|
||||
self.reset_count = 0
|
||||
self.update_calls: list[dict] = []
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
@@ -32,6 +33,9 @@ class _RecordingEngine:
|
||||
def finish_weight_update(self) -> None:
|
||||
self.finished = True
|
||||
|
||||
def reset_weight_update_target(self) -> None:
|
||||
self.reset_count += 1
|
||||
|
||||
|
||||
def _make_worker(engine: _RecordingEngine | None) -> Worker:
|
||||
worker = object.__new__(Worker)
|
||||
@@ -54,6 +58,7 @@ def test_start_update_finish_delegates_to_engine():
|
||||
|
||||
Worker.finish_weight_update(worker)
|
||||
assert engine.finished is True
|
||||
assert engine.reset_count == 1
|
||||
assert worker._weight_update_active is False
|
||||
|
||||
|
||||
@@ -85,6 +90,7 @@ def test_update_resets_active_on_error():
|
||||
Worker.update_weights(worker, {"names": ["w"]})
|
||||
|
||||
# A failed update ends the session so the next start is clean.
|
||||
assert engine.reset_count == 1
|
||||
assert worker._weight_update_active is False
|
||||
|
||||
|
||||
|
||||
@@ -73,6 +73,8 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
init_info_cls: type[TInitInfo]
|
||||
update_info_cls: type[TUpdateInfo]
|
||||
|
||||
supports_draft_weight_update: bool = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
@@ -95,6 +97,22 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
self.model_config = vllm_config.model_config
|
||||
self.device = device
|
||||
self.model = model
|
||||
self._default_model_config = self.model_config
|
||||
self._default_model = model
|
||||
|
||||
def set_weight_update_target(
|
||||
self,
|
||||
model: torch.nn.Module,
|
||||
model_config: Any,
|
||||
) -> None:
|
||||
"""Set the model that will receive the active weight update."""
|
||||
self.model = model
|
||||
self.model_config = model_config
|
||||
|
||||
def reset_weight_update_target(self) -> None:
|
||||
"""Restore weight updates to the engine's default target model."""
|
||||
self.model = self._default_model
|
||||
self.model_config = self._default_model_config
|
||||
|
||||
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
|
||||
"""
|
||||
|
||||
@@ -99,6 +99,7 @@ class SparseNCCLWeightTransferEngine(
|
||||
|
||||
init_info_cls = NCCLWeightTransferInitInfo
|
||||
update_info_cls = SparseNCCLWeightTransferUpdateInfo
|
||||
supports_draft_weight_update = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -248,6 +248,10 @@ class EngineClient(ABC):
|
||||
"""Start a new weight update."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def start_draft_weight_update(self) -> None:
|
||||
"""Start a new weight update targeting the speculative draft model."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def update_weights(self, request: WeightTransferUpdateRequest) -> None:
|
||||
"""Batched weight update for RL training."""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -877,6 +877,10 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
"""Start a new weight update."""
|
||||
self.llm_engine.collective_rpc("start_weight_update")
|
||||
|
||||
def start_draft_weight_update(self) -> None:
|
||||
"""Start a new weight update targeting the speculative draft model."""
|
||||
self.llm_engine.collective_rpc("start_draft_weight_update")
|
||||
|
||||
def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
|
||||
"""
|
||||
Update the weights of the model.
|
||||
|
||||
@@ -133,6 +133,12 @@ async def start_weight_update(raw_request: Request):
|
||||
return JSONResponse(content={"message": "Weight update started"})
|
||||
|
||||
|
||||
@router.post("/start_draft_weight_update")
|
||||
async def start_draft_weight_update(raw_request: Request):
|
||||
await engine_client(raw_request).start_draft_weight_update()
|
||||
return JSONResponse(content={"message": "Draft weight update started"})
|
||||
|
||||
|
||||
@router.post("/update_weights")
|
||||
async def update_weights(raw_request: Request):
|
||||
try:
|
||||
|
||||
@@ -1084,6 +1084,10 @@ class AsyncLLM(EngineClient):
|
||||
"""Start a new weight update."""
|
||||
await self.collective_rpc("start_weight_update")
|
||||
|
||||
async def start_draft_weight_update(self) -> None:
|
||||
"""Start a new weight update targeting the speculative draft model."""
|
||||
await self.collective_rpc("start_draft_weight_update")
|
||||
|
||||
async def update_weights(self, request: WeightTransferUpdateRequest) -> None:
|
||||
"""
|
||||
Batched weight update for RL training.
|
||||
|
||||
@@ -371,6 +371,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def get_model(self) -> nn.Module:
|
||||
return self.model
|
||||
|
||||
def get_draft_model(self) -> nn.Module | None:
|
||||
speculator = self.speculator
|
||||
if not isinstance(speculator, DraftModelSpeculator):
|
||||
return None
|
||||
return speculator.model
|
||||
|
||||
def reload_weights(self, *args, **kwargs) -> None:
|
||||
# TODO(Wentao): Use full version instead of import when fully migrated to v2
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1
|
||||
|
||||
@@ -3282,6 +3282,17 @@ class GPUModelRunner(
|
||||
return self.model.unwrap()
|
||||
return self.model
|
||||
|
||||
def get_draft_model(self) -> nn.Module | None:
|
||||
drafter = getattr(self, "drafter", None)
|
||||
if drafter is None:
|
||||
return None
|
||||
model = getattr(drafter, "model", None)
|
||||
if isinstance(
|
||||
model, (CUDAGraphWrapper, UBatchWrapper, BreakableCUDAGraphWrapper)
|
||||
):
|
||||
return cast(nn.Module, model.unwrap())
|
||||
return cast(nn.Module | None, model)
|
||||
|
||||
def get_supported_generation_tasks(self) -> list[GenerationTask]:
|
||||
model = self.get_model()
|
||||
supported_tasks = list[GenerationTask]()
|
||||
|
||||
@@ -154,6 +154,7 @@ class Worker(WorkerBase):
|
||||
|
||||
# Buffers saved before sleep
|
||||
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
|
||||
self._sleep_rebuild_draft_metadata_buffers = False
|
||||
|
||||
# Weight transfer engine is created in `load_model` once the model
|
||||
# is available, since the engine needs a reference to the model.
|
||||
@@ -198,6 +199,11 @@ class Worker(WorkerBase):
|
||||
self._sleep_saved_buffers = {
|
||||
name: buffer.cpu().clone() for name, buffer in model.named_buffers()
|
||||
}
|
||||
draft = self.get_draft_model()
|
||||
inner = getattr(draft, "model", None) if draft is not None else None
|
||||
self._sleep_rebuild_draft_metadata_buffers = inner is not None and hasattr(
|
||||
inner, "_build_fused_kv_buffers"
|
||||
)
|
||||
|
||||
self._get_sleep_mode_backend().suspend(level)
|
||||
|
||||
@@ -229,6 +235,14 @@ class Worker(WorkerBase):
|
||||
buffer.data.copy_(self._sleep_saved_buffers[name].data)
|
||||
self._sleep_saved_buffers = {}
|
||||
|
||||
if self._sleep_rebuild_draft_metadata_buffers:
|
||||
draft = self.get_draft_model()
|
||||
if draft is not None:
|
||||
inner = getattr(draft, "model", None)
|
||||
if inner is not None and hasattr(inner, "_build_fused_kv_buffers"):
|
||||
inner._build_fused_kv_buffers()
|
||||
self._sleep_rebuild_draft_metadata_buffers = False
|
||||
|
||||
if tags is None or "kv_cache" in tags:
|
||||
self.model_runner.post_kv_cache_wake_up()
|
||||
|
||||
@@ -915,6 +929,29 @@ class Worker(WorkerBase):
|
||||
def get_model(self) -> nn.Module:
|
||||
return self.model_runner.get_model()
|
||||
|
||||
def get_draft_model(self) -> nn.Module | None:
|
||||
return self.model_runner.get_draft_model()
|
||||
|
||||
def _set_draft_weight_update_target(self) -> None:
|
||||
assert self.weight_transfer_engine is not None
|
||||
|
||||
draft_model = self.get_draft_model()
|
||||
if draft_model is None:
|
||||
raise RuntimeError(
|
||||
"Draft model weight update requested, but no draft model is configured."
|
||||
)
|
||||
|
||||
speculative_config = self.speculative_config
|
||||
if speculative_config is None or speculative_config.draft_model_config is None:
|
||||
raise RuntimeError(
|
||||
"Draft model weight update requested, but no draft model "
|
||||
"config is configured."
|
||||
)
|
||||
|
||||
self.weight_transfer_engine.set_weight_update_target(
|
||||
draft_model, speculative_config.draft_model_config
|
||||
)
|
||||
|
||||
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
return self.model_runner.get_supported_tasks()
|
||||
|
||||
@@ -1179,16 +1216,38 @@ class Worker(WorkerBase):
|
||||
the configured weight transfer engine. The worker only tracks that a
|
||||
session is active.
|
||||
"""
|
||||
self._start_weight_update()
|
||||
|
||||
def start_draft_weight_update(self) -> None:
|
||||
"""
|
||||
Like start_weight_update, but retargets the engine at the speculative
|
||||
draft model for this session.
|
||||
"""
|
||||
self._start_weight_update(is_draft=True)
|
||||
|
||||
def _start_weight_update(self, is_draft: bool = False) -> None:
|
||||
self._check_weight_transfer_engine()
|
||||
assert self.weight_transfer_engine is not None
|
||||
|
||||
if is_draft and not self.weight_transfer_engine.supports_draft_weight_update:
|
||||
raise RuntimeError(
|
||||
f"{type(self.weight_transfer_engine).__name__} does not support "
|
||||
"draft model weight updates."
|
||||
)
|
||||
|
||||
if self._weight_update_active:
|
||||
raise RuntimeError(
|
||||
"start_weight_update called while a weight update is already "
|
||||
"active. Call finish_weight_update first."
|
||||
)
|
||||
|
||||
self.weight_transfer_engine.start_weight_update()
|
||||
try:
|
||||
if is_draft:
|
||||
self._set_draft_weight_update_target()
|
||||
self.weight_transfer_engine.start_weight_update()
|
||||
except BaseException:
|
||||
self.weight_transfer_engine.reset_weight_update_target()
|
||||
raise
|
||||
self._weight_update_active = True
|
||||
|
||||
def update_weights(self, update_info: dict) -> None:
|
||||
@@ -1197,6 +1256,8 @@ class Worker(WorkerBase):
|
||||
|
||||
start_weight_update must be called before update_weights and
|
||||
finish_weight_update must be called after all chunks have been sent.
|
||||
Every chunk loads into whichever model the session's start_weight_update
|
||||
/ start_draft_weight_update call selected.
|
||||
|
||||
Args:
|
||||
update_info: Dictionary containing backend-specific update info
|
||||
@@ -1213,6 +1274,7 @@ class Worker(WorkerBase):
|
||||
self.weight_transfer_engine.update_weights(update_info)
|
||||
except BaseException:
|
||||
self._weight_update_active = False
|
||||
self.weight_transfer_engine.reset_weight_update_target()
|
||||
raise
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
@@ -1226,6 +1288,7 @@ class Worker(WorkerBase):
|
||||
)
|
||||
|
||||
self.weight_transfer_engine.finish_weight_update()
|
||||
self.weight_transfer_engine.reset_weight_update_target()
|
||||
self._weight_update_active = False
|
||||
|
||||
def shutdown(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user