diff --git a/docs/training/rlhf.md b/docs/training/rlhf.md index 3eddd4fbec..2aae267e1d 100644 --- a/docs/training/rlhf.md +++ b/docs/training/rlhf.md @@ -16,7 +16,7 @@ The following open-source RL libraries use vLLM for fast rollouts (sorted alphab - [Unsloth](https://github.com/unslothai/unsloth) - [verl](https://github.com/volcengine/verl) -For weight synchronization between training and inference, see the [Weight Transfer](weight_transfer/README.md) documentation, which covers the pluggable backend system with [NCCL](weight_transfer/nccl.md) (multi-GPU) and [IPC](weight_transfer/ipc.md) (same-GPU) engines. +For weight synchronization between training and inference, see the [Weight Transfer](weight_transfer/README.md) documentation, which covers the pluggable backend system with [NCCL](weight_transfer/nccl.md) (multi-GPU) and [IPC](weight_transfer/ipc.md) (same-GPU) engines. To verify that an update landed on every rank, see the [Weight Checker](weight_checker.md). For pipelining generation and training to improve GPU utilization and throughput, see the [Async Reinforcement Learning](async_rl.md) guide, which covers the pause/resume API for safely updating weights mid-flight. diff --git a/docs/training/weight_checker.md b/docs/training/weight_checker.md new file mode 100644 index 0000000000..9936001f09 --- /dev/null +++ b/docs/training/weight_checker.md @@ -0,0 +1,60 @@ +# Weight Checker + +The Weight Checker verifies that an RL weight update landed on every rank. It +hashes each model parameter with SHA-256, so a caller can save the original +digests, reset the weights, transfer them back, and confirm they match. + +Enable it with `VLLM_SERVER_DEV_MODE=1`. All operations use +`POST /weight_checker`: + +| Request | Response | +| --- | --- | +| `{"action": "checksum"}` | `{"checksums": {key: sha256_hex}}` | +| `{"action": "reset"}` | `{"status": "reset"}` | +| `{"action": "compare", "baseline": {key: sha256_hex}}` | `{"match": bool, "mismatches": [key]}` | + +Keys have the form `dp{dp}:pp{pp}:pcp{pcp}:tp{tp}:{tensor_name}`, so each +shard is checked separately. All actions cover the target model's parameters, +which is what weight loading writes; `reset` zeroes them. The endpoint keeps no +state, so the caller holds the baseline. Invalid requests return HTTP 400. + +## RL weight-update check + +```bash +URL=http://localhost:8000 + +# 1. Stop serving, then save the original digests. +curl -X POST "$URL/pause?mode=abort" +curl -X POST $URL/weight_checker -H 'Content-Type: application/json' \ + -d '{"action":"checksum"}' > baseline.json + +# 2. Zero the weights. +curl -X POST $URL/weight_checker -H 'Content-Type: application/json' \ + -d '{"action":"reset"}' + +# 3. Transfer the original weights with start/update/finish_weight_update. + +# 4. Compare against the baseline; expect {"match": true, "mismatches": []}. +curl -X POST $URL/weight_checker -H 'Content-Type: application/json' \ + -d "{\"action\":\"compare\",\"baseline\":$(jq .checksums baseline.json)}" + +curl -X POST $URL/resume +``` + +## Limitations + +- Stay paused from `checksum` to `compare`: the weights are invalid between + `reset` and the transfer, and EPLB moves experts while serving. +- A request covers the engines managed by the API server it reaches. With + several API servers (for example `--data-parallel-external-lb`), send it to + each one. +- Buffers, draft models and LoRA adapters are not checked. +- CPU backends that repack linear weights hide them from the checker. +- compressed-tensors online transforms are not supported: their Hadamard + weights are normalized once per storage, so restored copies differ. +- MoE weights padded beyond the checkpoint shape (DeepEP hidden size, MXFP4) + keep uninitialized padding, so `compare` reports them as mismatches. +- The engine must be awake: sleep levels 1 and 2 release the weight memory. +- `--offload-backend prefetch` is not supported: parameters point at staging + buffers, not the offloaded weights. +- Hashing copies every weight to CPU, so keep it off latency-sensitive paths. diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 4611d0c6d6..28f3557324 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -18,6 +18,7 @@ mod server_info; mod sleep; mod tokenize; mod version; +mod weight_checker; mod weight_transfer; mod world_size; @@ -142,6 +143,7 @@ fn build_router_with_options( post(weight_transfer::update_weight_version), ) .route("/weight_info", get(weight_transfer::weight_info)) + .route("/weight_checker", post(weight_checker::weight_checker)) .route("/abort_requests", post(abort_requests::abort_requests)) .route("/sleep", post(sleep::sleep)) .route( diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 5c1fff2cae..b016810b59 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -5813,6 +5813,82 @@ async fn collective_rpc_route_sends_expected_utility_call_and_returns_results() ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn weight_checker_route_merges_worker_checksums_and_compares() { + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let workers = Value::Array(vec![ + Value::Map(vec![(Value::from("tp0:a"), Value::from("1"))]), + Value::Map(vec![(Value::from("tp1:a"), Value::from("2"))]), + ]); + for (method, result) in [ + ("compute_weight_checksums", workers.clone()), + ("compute_weight_checksums", workers), + ("reset_weights", Value::Array(vec![Value::Nil, Value::Nil])), + ] { + let utility = recv_engine_message(dealer).await; + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + assert_eq!(array[2], Value::from("collective_rpc")); + assert_eq!( + array[3].as_array().expect("rpc args")[0], + Value::from(method) + ); + let call_id = array[1].as_u64().expect("call id"); + let envelope = UtilityResultEnvelope::without_type_info(result); + send_outputs(push, utility_outputs(call_id, envelope)).await; + } + }) + }) + .await; + + let (status, body) = + post_json(&mut app, "/weight_checker", json!({"action": "checksum"})).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, json!({"checksums": {"tp0:a": "1", "tp1:a": "2"}})); + + let baseline = json!({"tp0:a": "1", "tp1:a": "9", "tp2:a": "3"}); + let (status, body) = post_json( + &mut app, + "/weight_checker", + json!({"action": "compare", "baseline": baseline}), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + body, + json!({"match": false, "mismatches": ["tp1:a", "tp2:a"]}) + ); + + let (status, body) = post_json(&mut app, "/weight_checker", json!({"action": "reset"})).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, json!({"status": "reset"})); + engine_task.abort_and_join().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn weight_checker_route_rejects_invalid_requests_before_engine_calls() { + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, _push| { + boxed_test_future(async move { + let message = recv_engine_message(dealer).await; + panic!("invalid request reached engine: {message:?}"); + }) + }) + .await; + for body in [ + json!({}), + json!({"action": "frobnicate"}), + json!({"action": "compare"}), + json!({"action": "compare", "baseline": {"k": 1}}), + ] { + let (status, _) = post_json(&mut app, "/weight_checker", body.clone()).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); + } + engine_task.abort_and_join().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn sleep_route_uses_python_compatible_default_query_values() { diff --git a/rust/src/server/src/routes/weight_checker.rs b/rust/src/server/src/routes/weight_checker.rs new file mode 100644 index 0000000000..af8baa06ed --- /dev/null +++ b/rust/src/server/src/routes/weight_checker.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use axum::Json; +use axum::extract::State; +use axum::extract::rejection::JsonRejection; +use serde::{Deserialize, Serialize}; +use serde_json::Value as JsonValue; + +use crate::error::{ApiError, invalid_request}; +use crate::state::AppState; +use crate::utils::utility_call_error; + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum WeightCheckerAction { + Checksum, + Reset, + Compare, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct WeightCheckerRequest { + action: WeightCheckerAction, + #[serde(default)] + baseline: Option>, +} + +#[derive(Serialize)] +#[serde(untagged)] +pub(crate) enum WeightCheckerResponse { + Checksums { + checksums: BTreeMap, + }, + Reset { + status: &'static str, + }, + Compare { + #[serde(rename = "match")] + matched: bool, + mismatches: Vec, + }, +} + +/// Checksum, reset, or compare model weights against a baseline. +pub async fn weight_checker( + State(state): State>, + body: Result, JsonRejection>, +) -> Result, ApiError> { + let Json(body) = body?; + let client = state.engine_core_client(); + let no_args = || { + ( + Vec::::new(), + BTreeMap::::new(), + ) + }; + + if body.action == WeightCheckerAction::Reset { + let (args, kwargs) = no_args(); + client + .collective_rpc("reset_weights", None, args, kwargs) + .await + .map_err(|error| utility_call_error("reset_weights", error))?; + return Ok(Json(WeightCheckerResponse::Reset { status: "reset" })); + } + if body.action == WeightCheckerAction::Compare && body.baseline.is_none() { + return Err(invalid_request!( + param = Some("baseline"), + "action='compare' requires a 'baseline' object" + )); + } + + let (args, kwargs) = no_args(); + let workers = client + .collective_rpc("compute_weight_checksums", None, args, kwargs) + .await + .map_err(|error| utility_call_error("compute_weight_checksums", error))?; + let mut checksums = BTreeMap::new(); + for worker in workers { + let worker: BTreeMap = rmpv::ext::from_value(worker).map_err(|error| { + ApiError::server_error(format!("invalid weight checksums: {error}")) + })?; + for (key, digest) in worker { + if checksums.contains_key(&key) { + return Err(ApiError::server_error(format!( + "Duplicate weight checksum keys: {key}" + ))); + } + checksums.insert(key, digest); + } + } + + let Some(baseline) = body.baseline.filter(|_| body.action == WeightCheckerAction::Compare) + else { + return Ok(Json(WeightCheckerResponse::Checksums { checksums })); + }; + let mismatches: Vec = checksums + .keys() + .chain(baseline.keys()) + .collect::>() + .into_iter() + .filter(|key| checksums.get(*key) != baseline.get(*key)) + .cloned() + .collect(); + Ok(Json(WeightCheckerResponse::Compare { + matched: mismatches.is_empty(), + mismatches, + })) +} diff --git a/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py b/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py new file mode 100644 index 0000000000..262b022ef5 --- /dev/null +++ b/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os +from unittest.mock import patch + +import pytest +import regex as re +import requests +import torch +from transformers import AutoModelForCausalLM + +from tests.entrypoints.serve.dev.rlhf.conftest import ( + MODEL_NAME, + gen, + health, + ok, + server, +) +from tests.utils import multi_gpu_test +from vllm.distributed.weight_transfer import ( + HTTPVLLMWeightSyncClient, + ModuleSource, + WeightTransferTrainerFactory, +) +from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo + + +def weight_checker(url: str, action: str, baseline=None) -> requests.Response: + body = {"action": action} + if baseline is not None: + body["baseline"] = baseline + return requests.post(f"{url}/weight_checker", json=body, timeout=300) + + +def checksums(url: str) -> dict[str, str]: + response = weight_checker(url, "checksum") + response.raise_for_status() + return response.json()["checksums"] + + +def ranks(keys) -> set[str]: + return {key.rsplit(":", 1)[0] for key in keys} + + +def reset_reload_and_compare(url: str) -> None: + baseline = checksums(url) + try: + weight_checker(url, "reset").raise_for_status() + comparison = weight_checker(url, "compare", baseline).json() + assert ranks(comparison["mismatches"]) == ranks(baseline) + finally: + requests.post( + f"{url}/collective_rpc", json={"method": "reload_weights"}, timeout=300 + ).raise_for_status() + assert weight_checker(url, "compare", baseline).json() == { + "match": True, + "mismatches": [], + } + + +@pytest.fixture(scope="class") +def server_url(): + with server() as url: + yield url + + +class TestSingleGPU: + @pytest.mark.parametrize( + "payload", + [{}, {"action": "frobnicate"}, {"action": "compare"}, [], None], + ) + def test_invalid_request_returns_400(self, server_url, payload): + response = requests.post( + f"{server_url}/weight_checker", json=payload, timeout=10 + ) + assert response.status_code == 400, response.text + assert health(server_url) == 200 + + def test_checksum_is_stable_and_compare_reports_differences(self, server_url): + baseline = checksums(server_url) + assert all( + re.fullmatch(r"[0-9a-f]{64}", digest) for digest in baseline.values() + ) + assert checksums(server_url) == baseline + + changed, missing, *_ = baseline + altered = {**baseline, changed: "0" * 64} + del altered[missing] + assert weight_checker(server_url, "compare", altered).json() == { + "match": False, + "mismatches": sorted([changed, missing]), + } + + def test_reset_changes_weights_and_reload_restores_them(self, server_url): + reset_reload_and_compare(server_url) + assert ok(gen(server_url)) + + +def test_ipc_weight_transfer_restores_reset_weights(): + """The documented RL flow: reset, then transfer the checkpoint back.""" + args = ["--weight-transfer-config", '{"backend": "ipc"}'] + with ( + patch.dict(os.environ, {"VLLM_ALLOW_INSECURE_SERIALIZATION": "1"}), + server(extra_args=args, port=8772) as url, + ): + baseline = checksums(url) + weight_checker(url, "reset").raise_for_status() + trainer_model = AutoModelForCausalLM.from_pretrained( + MODEL_NAME, torch_dtype=torch.bfloat16 + ).cuda() + WeightTransferTrainerFactory.trainer_init( + IPCTrainerInitInfo(rank=0), + client=HTTPVLLMWeightSyncClient(url), + source=ModuleSource(trainer_model), + ).send_weights() + assert weight_checker(url, "compare", baseline).json() == { + "match": True, + "mismatches": [], + } + + +@multi_gpu_test(num_gpus=2) +@pytest.mark.parametrize( + "args, expected", + [ + (["--data-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp1:pp0:pcp0:tp0"}), + (["--tensor-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp0:pp0:pcp0:tp1"}), + ], +) +def test_checksum_and_reset_cover_every_worker(args, expected): + with server(extra_args=args, port=8771, timeout=600) as url: + assert ranks(checksums(url)) == expected + reset_reload_and_compare(url) diff --git a/tests/model_executor/model_loader/test_weight_checksum.py b/tests/model_executor/model_loader/test_weight_checksum.py new file mode 100644 index 0000000000..e03d3b8814 --- /dev/null +++ b/tests/model_executor/model_loader/test_weight_checksum.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + +import pytest +import torch +import torch.nn as nn +from torch.testing._internal.two_tensor import TwoTensor + +from vllm.config import ParallelConfig +from vllm.model_executor import parameter +from vllm.model_executor.model_loader.weight_checksum import ( + compute_tensor_digests, + zero_weights, +) +from vllm.v1.worker import gpu_worker +from vllm.v1.worker.gpu_worker import Worker + +pytestmark = pytest.mark.cpu_test + + +class _Model(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(2, 2) + nn.init.ones_(self.linear.weight) + nn.init.ones_(self.linear.bias) + # A strided view, which cannot be reinterpreted as bytes in place. + self.strided = nn.Parameter(torch.arange(1.0, 7.0)[::2]) + self.scalar = nn.Parameter(torch.tensor(2.0)) + # Loading never restores buffers, so reset must leave them alone. + self.register_buffer("k_scale", torch.tensor(2.0)) + + +def test_digests_cover_every_parameter(): + assert set(compute_tensor_digests(_Model())) == { + "linear.weight", + "linear.bias", + "strided", + "scalar", + } + + +def test_zero_weights_changes_every_parameter_and_no_buffer(): + model = _Model() + before = compute_tensor_digests(model) + zero_weights(model) + after = compute_tensor_digests(model) + assert all(before[name] != after[name] for name in before) + assert model.k_scale.item() == 2.0 + + +def test_shared_weight_partitions_are_hashed_and_reset(monkeypatch): + monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1) + model = nn.Module() + model.transform = parameter.SharedWeightParameter(weight_loader=None) + model.transform.add_partition(0, object(), 2, 2) + model.transform.partitions[0].data.fill_(3.0) + + before = compute_tensor_digests(model) + zero_weights(model) + assert set(before) == {"transform.0"} + assert compute_tensor_digests(model) != before + + +def test_tensor_subclass_inner_tensors_are_hashed_and_reset(): + model = nn.Module() + model.weight = nn.Parameter( + TwoTensor(torch.ones(2, 2), torch.full((2, 2), 2.0)), requires_grad=False + ) + + before = compute_tensor_digests(model) + zero_weights(model) + assert set(before) == {"weight.a", "weight.b"} + assert all( + before[name] != digest for name, digest in compute_tensor_digests(model).items() + ) + + +def test_dense_dp_replicas_get_distinct_key_prefixes(monkeypatch): + for name in ("get_tp_group", "get_pp_group", "get_pcp_group"): + monkeypatch.setattr(gpu_worker, name, lambda: SimpleNamespace(rank_in_group=0)) + prefixes = [] + for dp_rank in (0, 1): + worker = object.__new__(Worker) + worker.parallel_config = ParallelConfig( + data_parallel_size=2, data_parallel_rank=dp_rank + ) + # Dense DP resets data_parallel_rank to 0 in each engine. + worker.parallel_config.reconfigure_for_independent_dp_rank() + prefixes.append(worker._weight_checksum_prefix()) + assert prefixes == ["dp0:pp0:pcp0:tp0:", "dp1:pp0:pcp0:tp0:"] diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index add424d32a..9bcc9d91eb 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -222,6 +222,24 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM return client +def test_dplb_weight_checksums_gather_every_engine(): + """call_utility_async on a DPLB client would keep only the first engine.""" + client = _make_dplb_client(num_engines=2) + + async def call_utility(method, *args, engine): + assert (method, args) == ("compute_weight_checksums", ()) + return [{f"dp{engine[0]}:tp{tp}:w": "x"} for tp in (0, 1)] + + client._call_utility_async = call_utility + workers = asyncio.run(client.compute_weight_checksums_async()) + assert [key for worker in workers for key in worker] == [ + "dp0:tp0:w", + "dp0:tp1:w", + "dp1:tp0:w", + "dp1:tp1:w", + ] + + def test_dplb_late_interaction_sticky_routing(): client = _make_dplb_client() diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 74849a85ce..71a61b1648 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -268,6 +268,10 @@ class EngineClient(ABC): """Perform a collective RPC call to the given path.""" raise NotImplementedError + async def compute_weight_checksums(self) -> list[dict[str, str]]: + """Return rank-qualified weight digests from every worker of every engine.""" + raise NotImplementedError + async def handle_fault( self, fault_tolerance_request: FaultToleranceRequest ) -> FaultToleranceResult: diff --git a/vllm/entrypoints/serve/dev/rlhf/api_router.py b/vllm/entrypoints/serve/dev/rlhf/api_router.py index 7af2ead22c..9c464e6a11 100644 --- a/vllm/entrypoints/serve/dev/rlhf/api_router.py +++ b/vllm/entrypoints/serve/dev/rlhf/api_router.py @@ -3,7 +3,7 @@ import json from http import HTTPStatus -from typing import Annotated, Any +from typing import Annotated, Any, Literal from fastapi import APIRouter, Body, FastAPI, HTTPException, Query, Request from fastapi.responses import JSONResponse @@ -216,6 +216,43 @@ async def weight_info(raw_request: Request): return JSONResponse(content={"weight_version": weight_version}) +@router.post("/weight_checker") +async def weight_checker( + raw_request: Request, + action: Annotated[Literal["checksum", "reset", "compare"], Body(embed=True)], + baseline: Annotated[dict[str, str] | None, Body(embed=True)] = None, +) -> JSONResponse: + """Checksum, reset, or compare model weights against a baseline.""" + client = engine_client(raw_request) + if action == "reset": + await client.collective_rpc("reset_weights") + return JSONResponse(content={"status": "reset"}) + if action == "compare" and baseline is None: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="action='compare' requires a 'baseline' object", + ) + + checksums: dict[str, str] = {} + for worker_checksums in await client.compute_weight_checksums(): + if duplicates := checksums.keys() & worker_checksums.keys(): + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + detail=f"Duplicate weight checksum keys: {sorted(duplicates)}", + ) + checksums.update(worker_checksums) + if action == "checksum": + return JSONResponse(content={"checksums": checksums}) + + assert baseline is not None + mismatches = sorted( + key + for key in checksums.keys() | baseline.keys() + if checksums.get(key) != baseline.get(key) + ) + return JSONResponse(content={"match": not mismatches, "mismatches": mismatches}) + + @router.get("/get_world_size") async def get_world_size( raw_request: Request, diff --git a/vllm/model_executor/model_loader/weight_checksum.py b/vllm/model_executor/model_loader/weight_checksum.py new file mode 100644 index 0000000000..40e62b0ed6 --- /dev/null +++ b/vllm/model_executor/model_loader/weight_checksum.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Per-parameter weight digests and reset, for verifying weight updates.""" + +import hashlib +from collections.abc import Iterator + +import torch +import torch.nn as nn +from torch.utils._python_dispatch import is_traceable_wrapper_subclass + +from vllm.model_executor.parameter import SharedWeightParameter + + +def _tensors(name: str, tensor: torch.Tensor) -> Iterator[tuple[str, torch.Tensor]]: + # Both keep their bytes in inner tensors (e.g. TorchAO's qdata and scale). + if isinstance(tensor, SharedWeightParameter): + for index, partition in tensor.partitions.items(): + yield from _tensors(f"{name}.{index}", partition) + elif is_traceable_wrapper_subclass(tensor): + for attr in tensor.__tensor_flatten__()[0]: + yield from _tensors(f"{name}.{attr}", getattr(tensor, attr)) + else: + yield name, tensor + + +def _weights(model: nn.Module) -> Iterator[tuple[str, torch.Tensor]]: + for name, param in model.named_parameters(): + yield from _tensors(name, param) + + +def compute_tensor_digests(model: nn.Module) -> dict[str, str]: + return { + name: hashlib.sha256( + weight.detach().cpu().contiguous().view(-1).view(torch.uint8).numpy() + ).hexdigest() + for name, weight in _weights(model) + } + + +def zero_weights(model: nn.Module) -> None: + for _, weight in _weights(model): + weight.data.zero_() diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 34f65f3ea2..ad9420247f 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -1125,6 +1125,9 @@ class AsyncLLM(EngineClient): async def is_sleeping(self) -> bool: return await self.engine_core.is_sleeping_async() + async def compute_weight_checksums(self) -> list[dict[str, str]]: + return await self.engine_core.compute_weight_checksums_async() + async def add_lora(self, lora_request: LoRARequest) -> bool: """Load a new LoRA adapter into the engine for future requests.""" return await self.engine_core.add_lora_async(lora_request) diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 5c09a0059e..cc0a8beddd 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -1005,6 +1005,9 @@ class EngineCore: def execute_dummy_batch(self): self.model_executor.execute_dummy_batch() + def compute_weight_checksums(self) -> list[dict[str, str]]: + return self.collective_rpc("compute_weight_checksums") + def add_lora(self, lora_request: LoRARequest) -> bool: return self.model_executor.add_lora(lora_request) diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index b03c40f440..9372c8a172 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -1362,6 +1362,10 @@ class AsyncMPClient(MPClient): "collective_rpc", method, timeout, args, kwargs ) + async def compute_weight_checksums_async(self) -> list[dict[str, str]]: + per_engine = await self.call_utility_all_async("compute_weight_checksums") + return [worker for workers in per_engine for worker in workers] + async def handle_fault( self, ft_request: FaultToleranceRequest ) -> FaultToleranceResult: diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index c6da90b077..8b7a0260a7 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -58,6 +58,10 @@ from vllm.distributed.weight_transfer import ( from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.model_executor.model_loader import get_model_loader +from vllm.model_executor.model_loader.weight_checksum import ( + compute_tensor_digests, + zero_weights, +) from vllm.model_executor.warmup.kernel_warmup import kernel_warmup from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory from vllm.platforms import current_platform @@ -342,6 +346,22 @@ class Worker(WorkerBase): def checkpoint_restore(self) -> None: checkpoint_restore_distributed_state() + def _weight_checksum_prefix(self) -> str: + # data_parallel_rank is 0 in dense workers; only the index is unique. + dp = self.parallel_config.data_parallel_index + pp = get_pp_group().rank_in_group + pcp = get_pcp_group().rank_in_group + tp = get_tp_group().rank_in_group + return f"dp{dp}:pp{pp}:pcp{pcp}:tp{tp}:" + + def compute_weight_checksums(self) -> dict[str, str]: + prefix = self._weight_checksum_prefix() + digests = compute_tensor_digests(self.model_runner.get_model()) + return {prefix + name: digest for name, digest in digests.items()} + + def reset_weights(self) -> None: + zero_weights(self.model_runner.get_model()) + def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager: if ( current_platform.is_cuda_alike()