Runtime Draft Weight Update for Speculative Decoding (#46725)

Signed-off-by: vx120 <893600387@qq.com>
Signed-off-by: vx120 <57470515+vx120@users.noreply.github.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: crp0128 <191679376@qq.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
vx120
2026-07-11 22:51:53 -07:00
committed by GitHub
co-authored by crp0128 aoshen02 Claude Opus 4.8
parent 481e481be7
commit fc1c548093
12 changed files with 129 additions and 1 deletions
+18
View File
@@ -73,6 +73,8 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
init_info_cls: type[TInitInfo]
update_info_cls: type[TUpdateInfo]
supports_draft_weight_update: bool = True
def __init__(
self,
config: WeightTransferConfig,
@@ -95,6 +97,22 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
self.model_config = vllm_config.model_config
self.device = device
self.model = model
self._default_model_config = self.model_config
self._default_model = model
def set_weight_update_target(
self,
model: torch.nn.Module,
model_config: Any,
) -> None:
"""Set the model that will receive the active weight update."""
self.model = model
self.model_config = model_config
def reset_weight_update_target(self) -> None:
"""Restore weight updates to the engine's default target model."""
self.model = self._default_model
self.model_config = self._default_model_config
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
"""
@@ -99,6 +99,7 @@ class SparseNCCLWeightTransferEngine(
init_info_cls = NCCLWeightTransferInitInfo
update_info_cls = SparseNCCLWeightTransferUpdateInfo
supports_draft_weight_update = False
def __init__(
self,