[feat] Add explicit /start_weight_update and /finish_weight_update APIs for weight transfer (#39212)

This commit is contained in:
Sumanth R Hegde
2026-05-08 18:03:33 -07:00
committed by GitHub
parent 30f519e947
commit e3b65a5ba0
19 changed files with 287 additions and 53 deletions
+2 -6
View File
@@ -4,7 +4,7 @@
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterator
from dataclasses import KW_ONLY, dataclass, field
from dataclasses import dataclass, field
from typing import Any, Generic, TypeVar
import torch
@@ -28,11 +28,7 @@ class WeightTransferInitInfo(ABC): # noqa: B024
class WeightTransferUpdateInfo(ABC): # noqa: B024
"""Base class for backend-specific weight update info."""
_: KW_ONLY
is_checkpoint_format: bool = True
"""Set to True if weights are in checkpoint/original model format and need
layerwise processing. Set to False if weights have already been processed
into kernel format (repacking, renaming, etc.)."""
pass
# API-level request classes (accept dicts for backend-agnostic serialization)
@@ -211,6 +211,11 @@ class IPCWeightTransferEngine(
- 'ray': Sends weights via Ray RPC to a Ray-based LLM handle
- 'http': Sends weights via HTTP POST to a vLLM HTTP server
.. note::
This method calls ``update_weights`` internally. The caller must
call ``start_weight_update`` before and ``finish_weight_update``
after this method.
Args:
iterator: Iterator of model parameters. Returns (name, tensor) tuples.
Tensors should be on the same GPU as the inference workers.