[RL] Add rank-local IPC weight updates (#52497)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
aoshen02
2026-08-26 15:36:57 +08:00
committed by GitHub
co-authored by OpenAI Codex OpenAI Codex Roger Wang
parent a5f82882a6
commit ee5ae2ae70
6 changed files with 80 additions and 8 deletions
+3 -2
View File
@@ -27,6 +27,7 @@ from vllm.config.weight_transfer import WeightTransferConfig
TInitInfo = TypeVar("TInitInfo", bound="WeightTransferInitInfo")
TUpdateInfo = TypeVar("TUpdateInfo", bound="WeightTransferUpdateInfo")
TTrainerInitInfo = TypeVar("TTrainerInitInfo", bound="TrainerInitInfo")
WeightTransferUpdatePayload = dict[str, Any] | list[dict[str, Any]]
# A trainer supplies its parameters as a `WeightSource` (defined below): a
# re-iterable stream of materialized `(name, tensor)` pairs plus a `metadata()`
@@ -324,7 +325,7 @@ class WeightTransferInitRequest:
class WeightTransferUpdateRequest:
"""API-level weight update request."""
update_info: dict[str, Any] = field(default_factory=dict)
update_info: WeightTransferUpdatePayload = field(default_factory=dict)
class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
@@ -545,7 +546,7 @@ class VLLMWeightSyncClient(Protocol):
def start_weight_update(self) -> None: ...
def update_weights(self, update_info: dict[str, Any]) -> None: ...
def update_weights(self, update_info: WeightTransferUpdatePayload) -> None: ...
def finish_weight_update(self, weight_version: str | None = None) -> None: ...
+12 -3
View File
@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any
from vllm.distributed.weight_transfer.base import (
WeightTransferInitRequest,
WeightTransferUpdatePayload,
WeightTransferUpdateRequest,
)
@@ -46,6 +47,14 @@ def _json_safe_update_info(update_info: dict[str, Any]) -> dict[str, Any]:
return out
def _json_safe_update_payload(
update_info: WeightTransferUpdatePayload,
) -> WeightTransferUpdatePayload:
if isinstance(update_info, list):
return [_json_safe_update_info(info) for info in update_info]
return _json_safe_update_info(update_info)
class HTTPVLLMWeightSyncClient:
"""Talks to a vLLM server over the RLHF HTTP routes.
@@ -72,9 +81,9 @@ class HTTPVLLMWeightSyncClient:
def start_weight_update(self) -> None:
self._post("start_weight_update")
def update_weights(self, update_info: dict[str, Any]) -> None:
def update_weights(self, update_info: WeightTransferUpdatePayload) -> None:
self._post(
"update_weights", {"update_info": _json_safe_update_info(update_info)}
"update_weights", {"update_info": _json_safe_update_payload(update_info)}
)
def finish_weight_update(self, weight_version: str | None = None) -> None:
@@ -105,7 +114,7 @@ class RayVLLMWeightSyncClient:
ray.get([h.start_weight_update.remote() for h in self.handles])
def update_weights(self, update_info: dict[str, Any]) -> None:
def update_weights(self, update_info: WeightTransferUpdatePayload) -> None:
import ray
request = WeightTransferUpdateRequest(update_info=update_info)