diff --git a/docs/training/weight_transfer/base.md b/docs/training/weight_transfer/base.md index 0208264966..f69af8dd6e 100644 --- a/docs/training/weight_transfer/base.md +++ b/docs/training/weight_transfer/base.md @@ -49,6 +49,10 @@ update_request = WeightTransferUpdateRequest( ) ``` +At the LLM/API layer, call `start_draft_weight_update()` instead of +`start_weight_update()` to target the speculative draft model; +`update_weights` / `finish_weight_update` are unchanged. + ### WeightTransferUpdateInfo The base `WeightTransferUpdateInfo` is a marker class for backend-specific update info: diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index 38ea2661c8..2985c53951 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -145,6 +145,7 @@ def test_openapi_stateless(case: schemathesis.Case): if case.operation.path in ( "/init_weight_transfer_engine", "/start_weight_update", + "/start_draft_weight_update", "/update_weights", "/finish_weight_update", ): diff --git a/tests/v1/worker/test_gpu_worker_weight_transfer.py b/tests/v1/worker/test_gpu_worker_weight_transfer.py index 93c2e916b1..aeb727d9ce 100644 --- a/tests/v1/worker/test_gpu_worker_weight_transfer.py +++ b/tests/v1/worker/test_gpu_worker_weight_transfer.py @@ -19,6 +19,7 @@ class _RecordingEngine: self.raise_on_update = raise_on_update self.started = False self.finished = False + self.reset_count = 0 self.update_calls: list[dict] = [] def start_weight_update(self) -> None: @@ -32,6 +33,9 @@ class _RecordingEngine: def finish_weight_update(self) -> None: self.finished = True + def reset_weight_update_target(self) -> None: + self.reset_count += 1 + def _make_worker(engine: _RecordingEngine | None) -> Worker: worker = object.__new__(Worker) @@ -54,6 +58,7 @@ def test_start_update_finish_delegates_to_engine(): Worker.finish_weight_update(worker) assert engine.finished is True + assert engine.reset_count == 1 assert worker._weight_update_active is False @@ -85,6 +90,7 @@ def test_update_resets_active_on_error(): Worker.update_weights(worker, {"names": ["w"]}) # A failed update ends the session so the next start is clean. + assert engine.reset_count == 1 assert worker._weight_update_active is False diff --git a/vllm/distributed/weight_transfer/base.py b/vllm/distributed/weight_transfer/base.py index 5ed99ec546..6dbd768d25 100644 --- a/vllm/distributed/weight_transfer/base.py +++ b/vllm/distributed/weight_transfer/base.py @@ -73,6 +73,8 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): init_info_cls: type[TInitInfo] update_info_cls: type[TUpdateInfo] + supports_draft_weight_update: bool = True + def __init__( self, config: WeightTransferConfig, @@ -95,6 +97,22 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): self.model_config = vllm_config.model_config self.device = device self.model = model + self._default_model_config = self.model_config + self._default_model = model + + def set_weight_update_target( + self, + model: torch.nn.Module, + model_config: Any, + ) -> None: + """Set the model that will receive the active weight update.""" + self.model = model + self.model_config = model_config + + def reset_weight_update_target(self) -> None: + """Restore weight updates to the engine's default target model.""" + self.model = self._default_model + self.model_config = self._default_model_config def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo: """ diff --git a/vllm/distributed/weight_transfer/sparse_nccl_engine.py b/vllm/distributed/weight_transfer/sparse_nccl_engine.py index 669b066a8e..2666afcf4e 100644 --- a/vllm/distributed/weight_transfer/sparse_nccl_engine.py +++ b/vllm/distributed/weight_transfer/sparse_nccl_engine.py @@ -99,6 +99,7 @@ class SparseNCCLWeightTransferEngine( init_info_cls = NCCLWeightTransferInitInfo update_info_cls = SparseNCCLWeightTransferUpdateInfo + supports_draft_weight_update = False def __init__( self, diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 7d5cc164f0..c54123bea9 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -248,6 +248,10 @@ class EngineClient(ABC): """Start a new weight update.""" raise NotImplementedError + async def start_draft_weight_update(self) -> None: + """Start a new weight update targeting the speculative draft model.""" + raise NotImplementedError + async def update_weights(self, request: WeightTransferUpdateRequest) -> None: """Batched weight update for RL training.""" raise NotImplementedError diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index a3ed94ee0a..014a670061 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -877,6 +877,10 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): """Start a new weight update.""" self.llm_engine.collective_rpc("start_weight_update") + def start_draft_weight_update(self) -> None: + """Start a new weight update targeting the speculative draft model.""" + self.llm_engine.collective_rpc("start_draft_weight_update") + def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None: """ Update the weights of the model. diff --git a/vllm/entrypoints/serve/dev/rlhf/api_router.py b/vllm/entrypoints/serve/dev/rlhf/api_router.py index 310e4021eb..9e9fc0d3d8 100644 --- a/vllm/entrypoints/serve/dev/rlhf/api_router.py +++ b/vllm/entrypoints/serve/dev/rlhf/api_router.py @@ -133,6 +133,12 @@ async def start_weight_update(raw_request: Request): return JSONResponse(content={"message": "Weight update started"}) +@router.post("/start_draft_weight_update") +async def start_draft_weight_update(raw_request: Request): + await engine_client(raw_request).start_draft_weight_update() + return JSONResponse(content={"message": "Draft weight update started"}) + + @router.post("/update_weights") async def update_weights(raw_request: Request): try: diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 61f02092bd..8bcd4ba89a 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -1084,6 +1084,10 @@ class AsyncLLM(EngineClient): """Start a new weight update.""" await self.collective_rpc("start_weight_update") + async def start_draft_weight_update(self) -> None: + """Start a new weight update targeting the speculative draft model.""" + await self.collective_rpc("start_draft_weight_update") + async def update_weights(self, request: WeightTransferUpdateRequest) -> None: """ Batched weight update for RL training. diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 0a62295629..d2e1e0524f 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -371,6 +371,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): def get_model(self) -> nn.Module: return self.model + def get_draft_model(self) -> nn.Module | None: + speculator = self.speculator + if not isinstance(speculator, DraftModelSpeculator): + return None + return speculator.model + def reload_weights(self, *args, **kwargs) -> None: # TODO(Wentao): Use full version instead of import when fully migrated to v2 from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1 diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index cb8ed5f367..a675fcfffe 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -3282,6 +3282,17 @@ class GPUModelRunner( return self.model.unwrap() return self.model + def get_draft_model(self) -> nn.Module | None: + drafter = getattr(self, "drafter", None) + if drafter is None: + return None + model = getattr(drafter, "model", None) + if isinstance( + model, (CUDAGraphWrapper, UBatchWrapper, BreakableCUDAGraphWrapper) + ): + return cast(nn.Module, model.unwrap()) + return cast(nn.Module | None, model) + def get_supported_generation_tasks(self) -> list[GenerationTask]: model = self.get_model() supported_tasks = list[GenerationTask]() diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 1cd7e2598a..5fb0c38773 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -154,6 +154,7 @@ class Worker(WorkerBase): # Buffers saved before sleep self._sleep_saved_buffers: dict[str, torch.Tensor] = {} + self._sleep_rebuild_draft_metadata_buffers = False # Weight transfer engine is created in `load_model` once the model # is available, since the engine needs a reference to the model. @@ -198,6 +199,11 @@ class Worker(WorkerBase): self._sleep_saved_buffers = { name: buffer.cpu().clone() for name, buffer in model.named_buffers() } + draft = self.get_draft_model() + inner = getattr(draft, "model", None) if draft is not None else None + self._sleep_rebuild_draft_metadata_buffers = inner is not None and hasattr( + inner, "_build_fused_kv_buffers" + ) self._get_sleep_mode_backend().suspend(level) @@ -229,6 +235,14 @@ class Worker(WorkerBase): buffer.data.copy_(self._sleep_saved_buffers[name].data) self._sleep_saved_buffers = {} + if self._sleep_rebuild_draft_metadata_buffers: + draft = self.get_draft_model() + if draft is not None: + inner = getattr(draft, "model", None) + if inner is not None and hasattr(inner, "_build_fused_kv_buffers"): + inner._build_fused_kv_buffers() + self._sleep_rebuild_draft_metadata_buffers = False + if tags is None or "kv_cache" in tags: self.model_runner.post_kv_cache_wake_up() @@ -915,6 +929,29 @@ class Worker(WorkerBase): def get_model(self) -> nn.Module: return self.model_runner.get_model() + def get_draft_model(self) -> nn.Module | None: + return self.model_runner.get_draft_model() + + def _set_draft_weight_update_target(self) -> None: + assert self.weight_transfer_engine is not None + + draft_model = self.get_draft_model() + if draft_model is None: + raise RuntimeError( + "Draft model weight update requested, but no draft model is configured." + ) + + speculative_config = self.speculative_config + if speculative_config is None or speculative_config.draft_model_config is None: + raise RuntimeError( + "Draft model weight update requested, but no draft model " + "config is configured." + ) + + self.weight_transfer_engine.set_weight_update_target( + draft_model, speculative_config.draft_model_config + ) + def get_supported_tasks(self) -> tuple[SupportedTask, ...]: return self.model_runner.get_supported_tasks() @@ -1179,16 +1216,38 @@ class Worker(WorkerBase): the configured weight transfer engine. The worker only tracks that a session is active. """ + self._start_weight_update() + + def start_draft_weight_update(self) -> None: + """ + Like start_weight_update, but retargets the engine at the speculative + draft model for this session. + """ + self._start_weight_update(is_draft=True) + + def _start_weight_update(self, is_draft: bool = False) -> None: self._check_weight_transfer_engine() assert self.weight_transfer_engine is not None + if is_draft and not self.weight_transfer_engine.supports_draft_weight_update: + raise RuntimeError( + f"{type(self.weight_transfer_engine).__name__} does not support " + "draft model weight updates." + ) + if self._weight_update_active: raise RuntimeError( "start_weight_update called while a weight update is already " "active. Call finish_weight_update first." ) - self.weight_transfer_engine.start_weight_update() + try: + if is_draft: + self._set_draft_weight_update_target() + self.weight_transfer_engine.start_weight_update() + except BaseException: + self.weight_transfer_engine.reset_weight_update_target() + raise self._weight_update_active = True def update_weights(self, update_info: dict) -> None: @@ -1197,6 +1256,8 @@ class Worker(WorkerBase): start_weight_update must be called before update_weights and finish_weight_update must be called after all chunks have been sent. + Every chunk loads into whichever model the session's start_weight_update + / start_draft_weight_update call selected. Args: update_info: Dictionary containing backend-specific update info @@ -1213,6 +1274,7 @@ class Worker(WorkerBase): self.weight_transfer_engine.update_weights(update_info) except BaseException: self._weight_update_active = False + self.weight_transfer_engine.reset_weight_update_target() raise def finish_weight_update(self) -> None: @@ -1226,6 +1288,7 @@ class Worker(WorkerBase): ) self.weight_transfer_engine.finish_weight_update() + self.weight_transfer_engine.reset_weight_update_target() self._weight_update_active = False def shutdown(self) -> None: