support rl feature : weight checker (#51350)
Signed-off-by: shiyuan680 <yangcheng104@huawei.com> Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: shiyuan680 <yangcheng104@huawei.com> Co-authored-by: Ronald1995 <ronaldautomobile@163.com> Co-authored-by: DeepSeek Harness <noreply@deepseek.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
shiyuan680
Ronald1995
DeepSeek Harness
aoshen02
Claude Opus 5.5
aoshen02
parent
a8e069f81b
commit
42d9f8ccdb
@@ -0,0 +1,133 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
import requests
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
from tests.entrypoints.serve.dev.rlhf.conftest import (
|
||||
MODEL_NAME,
|
||||
gen,
|
||||
health,
|
||||
ok,
|
||||
server,
|
||||
)
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
||||
|
||||
|
||||
def weight_checker(url: str, action: str, baseline=None) -> requests.Response:
|
||||
body = {"action": action}
|
||||
if baseline is not None:
|
||||
body["baseline"] = baseline
|
||||
return requests.post(f"{url}/weight_checker", json=body, timeout=300)
|
||||
|
||||
|
||||
def checksums(url: str) -> dict[str, str]:
|
||||
response = weight_checker(url, "checksum")
|
||||
response.raise_for_status()
|
||||
return response.json()["checksums"]
|
||||
|
||||
|
||||
def ranks(keys) -> set[str]:
|
||||
return {key.rsplit(":", 1)[0] for key in keys}
|
||||
|
||||
|
||||
def reset_reload_and_compare(url: str) -> None:
|
||||
baseline = checksums(url)
|
||||
try:
|
||||
weight_checker(url, "reset").raise_for_status()
|
||||
comparison = weight_checker(url, "compare", baseline).json()
|
||||
assert ranks(comparison["mismatches"]) == ranks(baseline)
|
||||
finally:
|
||||
requests.post(
|
||||
f"{url}/collective_rpc", json={"method": "reload_weights"}, timeout=300
|
||||
).raise_for_status()
|
||||
assert weight_checker(url, "compare", baseline).json() == {
|
||||
"match": True,
|
||||
"mismatches": [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def server_url():
|
||||
with server() as url:
|
||||
yield url
|
||||
|
||||
|
||||
class TestSingleGPU:
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[{}, {"action": "frobnicate"}, {"action": "compare"}, [], None],
|
||||
)
|
||||
def test_invalid_request_returns_400(self, server_url, payload):
|
||||
response = requests.post(
|
||||
f"{server_url}/weight_checker", json=payload, timeout=10
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
assert health(server_url) == 200
|
||||
|
||||
def test_checksum_is_stable_and_compare_reports_differences(self, server_url):
|
||||
baseline = checksums(server_url)
|
||||
assert all(
|
||||
re.fullmatch(r"[0-9a-f]{64}", digest) for digest in baseline.values()
|
||||
)
|
||||
assert checksums(server_url) == baseline
|
||||
|
||||
changed, missing, *_ = baseline
|
||||
altered = {**baseline, changed: "0" * 64}
|
||||
del altered[missing]
|
||||
assert weight_checker(server_url, "compare", altered).json() == {
|
||||
"match": False,
|
||||
"mismatches": sorted([changed, missing]),
|
||||
}
|
||||
|
||||
def test_reset_changes_weights_and_reload_restores_them(self, server_url):
|
||||
reset_reload_and_compare(server_url)
|
||||
assert ok(gen(server_url))
|
||||
|
||||
|
||||
def test_ipc_weight_transfer_restores_reset_weights():
|
||||
"""The documented RL flow: reset, then transfer the checkpoint back."""
|
||||
args = ["--weight-transfer-config", '{"backend": "ipc"}']
|
||||
with (
|
||||
patch.dict(os.environ, {"VLLM_ALLOW_INSECURE_SERIALIZATION": "1"}),
|
||||
server(extra_args=args, port=8772) as url,
|
||||
):
|
||||
baseline = checksums(url)
|
||||
weight_checker(url, "reset").raise_for_status()
|
||||
trainer_model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_NAME, torch_dtype=torch.bfloat16
|
||||
).cuda()
|
||||
WeightTransferTrainerFactory.trainer_init(
|
||||
IPCTrainerInitInfo(rank=0),
|
||||
client=HTTPVLLMWeightSyncClient(url),
|
||||
source=ModuleSource(trainer_model),
|
||||
).send_weights()
|
||||
assert weight_checker(url, "compare", baseline).json() == {
|
||||
"match": True,
|
||||
"mismatches": [],
|
||||
}
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"args, expected",
|
||||
[
|
||||
(["--data-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp1:pp0:pcp0:tp0"}),
|
||||
(["--tensor-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp0:pp0:pcp0:tp1"}),
|
||||
],
|
||||
)
|
||||
def test_checksum_and_reset_cover_every_worker(args, expected):
|
||||
with server(extra_args=args, port=8771, timeout=600) as url:
|
||||
assert ranks(checksums(url)) == expected
|
||||
reset_reload_and_compare(url)
|
||||
@@ -0,0 +1,93 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.testing._internal.two_tensor import TwoTensor
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.model_executor import parameter
|
||||
from vllm.model_executor.model_loader.weight_checksum import (
|
||||
compute_tensor_digests,
|
||||
zero_weights,
|
||||
)
|
||||
from vllm.v1.worker import gpu_worker
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _Model(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.linear = nn.Linear(2, 2)
|
||||
nn.init.ones_(self.linear.weight)
|
||||
nn.init.ones_(self.linear.bias)
|
||||
# A strided view, which cannot be reinterpreted as bytes in place.
|
||||
self.strided = nn.Parameter(torch.arange(1.0, 7.0)[::2])
|
||||
self.scalar = nn.Parameter(torch.tensor(2.0))
|
||||
# Loading never restores buffers, so reset must leave them alone.
|
||||
self.register_buffer("k_scale", torch.tensor(2.0))
|
||||
|
||||
|
||||
def test_digests_cover_every_parameter():
|
||||
assert set(compute_tensor_digests(_Model())) == {
|
||||
"linear.weight",
|
||||
"linear.bias",
|
||||
"strided",
|
||||
"scalar",
|
||||
}
|
||||
|
||||
|
||||
def test_zero_weights_changes_every_parameter_and_no_buffer():
|
||||
model = _Model()
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
after = compute_tensor_digests(model)
|
||||
assert all(before[name] != after[name] for name in before)
|
||||
assert model.k_scale.item() == 2.0
|
||||
|
||||
|
||||
def test_shared_weight_partitions_are_hashed_and_reset(monkeypatch):
|
||||
monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
model = nn.Module()
|
||||
model.transform = parameter.SharedWeightParameter(weight_loader=None)
|
||||
model.transform.add_partition(0, object(), 2, 2)
|
||||
model.transform.partitions[0].data.fill_(3.0)
|
||||
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
assert set(before) == {"transform.0"}
|
||||
assert compute_tensor_digests(model) != before
|
||||
|
||||
|
||||
def test_tensor_subclass_inner_tensors_are_hashed_and_reset():
|
||||
model = nn.Module()
|
||||
model.weight = nn.Parameter(
|
||||
TwoTensor(torch.ones(2, 2), torch.full((2, 2), 2.0)), requires_grad=False
|
||||
)
|
||||
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
assert set(before) == {"weight.a", "weight.b"}
|
||||
assert all(
|
||||
before[name] != digest for name, digest in compute_tensor_digests(model).items()
|
||||
)
|
||||
|
||||
|
||||
def test_dense_dp_replicas_get_distinct_key_prefixes(monkeypatch):
|
||||
for name in ("get_tp_group", "get_pp_group", "get_pcp_group"):
|
||||
monkeypatch.setattr(gpu_worker, name, lambda: SimpleNamespace(rank_in_group=0))
|
||||
prefixes = []
|
||||
for dp_rank in (0, 1):
|
||||
worker = object.__new__(Worker)
|
||||
worker.parallel_config = ParallelConfig(
|
||||
data_parallel_size=2, data_parallel_rank=dp_rank
|
||||
)
|
||||
# Dense DP resets data_parallel_rank to 0 in each engine.
|
||||
worker.parallel_config.reconfigure_for_independent_dp_rank()
|
||||
prefixes.append(worker._weight_checksum_prefix())
|
||||
assert prefixes == ["dp0:pp0:pcp0:tp0:", "dp1:pp0:pcp0:tp0:"]
|
||||
@@ -222,6 +222,24 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM
|
||||
return client
|
||||
|
||||
|
||||
def test_dplb_weight_checksums_gather_every_engine():
|
||||
"""call_utility_async on a DPLB client would keep only the first engine."""
|
||||
client = _make_dplb_client(num_engines=2)
|
||||
|
||||
async def call_utility(method, *args, engine):
|
||||
assert (method, args) == ("compute_weight_checksums", ())
|
||||
return [{f"dp{engine[0]}:tp{tp}:w": "x"} for tp in (0, 1)]
|
||||
|
||||
client._call_utility_async = call_utility
|
||||
workers = asyncio.run(client.compute_weight_checksums_async())
|
||||
assert [key for worker in workers for key in worker] == [
|
||||
"dp0:tp0:w",
|
||||
"dp0:tp1:w",
|
||||
"dp1:tp0:w",
|
||||
"dp1:tp1:w",
|
||||
]
|
||||
|
||||
|
||||
def test_dplb_late_interaction_sticky_routing():
|
||||
client = _make_dplb_client()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user