[feat] Add explicit /start_weight_update and /finish_weight_update APIs for weight transfer (#39212)
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import KW_ONLY, dataclass, field
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Generic, TypeVar
|
||||
|
||||
import torch
|
||||
@@ -28,11 +28,7 @@ class WeightTransferInitInfo(ABC): # noqa: B024
|
||||
class WeightTransferUpdateInfo(ABC): # noqa: B024
|
||||
"""Base class for backend-specific weight update info."""
|
||||
|
||||
_: KW_ONLY
|
||||
is_checkpoint_format: bool = True
|
||||
"""Set to True if weights are in checkpoint/original model format and need
|
||||
layerwise processing. Set to False if weights have already been processed
|
||||
into kernel format (repacking, renaming, etc.)."""
|
||||
pass
|
||||
|
||||
|
||||
# API-level request classes (accept dicts for backend-agnostic serialization)
|
||||
|
||||
@@ -211,6 +211,11 @@ class IPCWeightTransferEngine(
|
||||
- 'ray': Sends weights via Ray RPC to a Ray-based LLM handle
|
||||
- 'http': Sends weights via HTTP POST to a vLLM HTTP server
|
||||
|
||||
.. note::
|
||||
This method calls ``update_weights`` internally. The caller must
|
||||
call ``start_weight_update`` before and ``finish_weight_update``
|
||||
after this method.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of model parameters. Returns (name, tensor) tuples.
|
||||
Tensors should be on the same GPU as the inference workers.
|
||||
|
||||
Reference in New Issue
Block a user