[RL] Add rank-local IPC weight updates (#52497)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
OpenAI Codex
OpenAI Codex
Roger Wang
parent
a5f82882a6
commit
ee5ae2ae70
@@ -27,6 +27,7 @@ from vllm.config.weight_transfer import WeightTransferConfig
|
||||
TInitInfo = TypeVar("TInitInfo", bound="WeightTransferInitInfo")
|
||||
TUpdateInfo = TypeVar("TUpdateInfo", bound="WeightTransferUpdateInfo")
|
||||
TTrainerInitInfo = TypeVar("TTrainerInitInfo", bound="TrainerInitInfo")
|
||||
WeightTransferUpdatePayload = dict[str, Any] | list[dict[str, Any]]
|
||||
|
||||
# A trainer supplies its parameters as a `WeightSource` (defined below): a
|
||||
# re-iterable stream of materialized `(name, tensor)` pairs plus a `metadata()`
|
||||
@@ -324,7 +325,7 @@ class WeightTransferInitRequest:
|
||||
class WeightTransferUpdateRequest:
|
||||
"""API-level weight update request."""
|
||||
|
||||
update_info: dict[str, Any] = field(default_factory=dict)
|
||||
update_info: WeightTransferUpdatePayload = field(default_factory=dict)
|
||||
|
||||
|
||||
class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
@@ -545,7 +546,7 @@ class VLLMWeightSyncClient(Protocol):
|
||||
|
||||
def start_weight_update(self) -> None: ...
|
||||
|
||||
def update_weights(self, update_info: dict[str, Any]) -> None: ...
|
||||
def update_weights(self, update_info: WeightTransferUpdatePayload) -> None: ...
|
||||
|
||||
def finish_weight_update(self, weight_version: str | None = None) -> None: ...
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdatePayload,
|
||||
WeightTransferUpdateRequest,
|
||||
)
|
||||
|
||||
@@ -46,6 +47,14 @@ def _json_safe_update_info(update_info: dict[str, Any]) -> dict[str, Any]:
|
||||
return out
|
||||
|
||||
|
||||
def _json_safe_update_payload(
|
||||
update_info: WeightTransferUpdatePayload,
|
||||
) -> WeightTransferUpdatePayload:
|
||||
if isinstance(update_info, list):
|
||||
return [_json_safe_update_info(info) for info in update_info]
|
||||
return _json_safe_update_info(update_info)
|
||||
|
||||
|
||||
class HTTPVLLMWeightSyncClient:
|
||||
"""Talks to a vLLM server over the RLHF HTTP routes.
|
||||
|
||||
@@ -72,9 +81,9 @@ class HTTPVLLMWeightSyncClient:
|
||||
def start_weight_update(self) -> None:
|
||||
self._post("start_weight_update")
|
||||
|
||||
def update_weights(self, update_info: dict[str, Any]) -> None:
|
||||
def update_weights(self, update_info: WeightTransferUpdatePayload) -> None:
|
||||
self._post(
|
||||
"update_weights", {"update_info": _json_safe_update_info(update_info)}
|
||||
"update_weights", {"update_info": _json_safe_update_payload(update_info)}
|
||||
)
|
||||
|
||||
def finish_weight_update(self, weight_version: str | None = None) -> None:
|
||||
@@ -105,7 +114,7 @@ class RayVLLMWeightSyncClient:
|
||||
|
||||
ray.get([h.start_weight_update.remote() for h in self.handles])
|
||||
|
||||
def update_weights(self, update_info: dict[str, Any]) -> None:
|
||||
def update_weights(self, update_info: WeightTransferUpdatePayload) -> None:
|
||||
import ray
|
||||
|
||||
request = WeightTransferUpdateRequest(update_info=update_info)
|
||||
|
||||
Reference in New Issue
Block a user