From d09ff7763747bbe7cd153edd8017479804f47be7 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Tue, 6 Oct 2026 01:38:18 +0800 Subject: [PATCH] [Bugfix] Give each data-parallel engine its own global RNG streams (#59788) Signed-off-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- tests/utils_/test_torch_utils.py | 15 +++++++++++++++ vllm/utils/torch_utils.py | 6 +++++- vllm/v1/worker/cpu_worker.py | 4 +++- vllm/v1/worker/gpu_worker.py | 4 +++- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/utils_/test_torch_utils.py b/tests/utils_/test_torch_utils.py index 79dd3e320a..7294253462 100644 --- a/tests/utils_/test_torch_utils.py +++ b/tests/utils_/test_torch_utils.py @@ -14,6 +14,7 @@ from vllm.utils.torch_utils import ( is_lossless_cast, is_quantized_kv_cache, set_default_torch_dtype, + set_random_seed, set_torch_threads_for_runtime, startup_omp_num_threads, ) @@ -217,3 +218,17 @@ def test_async_tensor_h2d_staging(device): async_tensor_h2d([1, 2, 3], device=device, dtype=torch.int32), torch.tensor([1, 2, 3], dtype=torch.int32, device=device), ) + + +def test_set_random_seed_differs_per_data_parallel_index(): + """Unseeded requests draw from the global RNGs; DP engines sharing the engine + seed must get distinct, reproducible streams, and index 0 keeps the seed.""" + + def draws(*args): + set_random_seed(*args) + return np.random.randint(2**31, size=4).tolist(), torch.rand(4).tolist() + + assert draws(42, 0) == draws(42) + assert draws(42, 1) == draws(42, 1) + assert len({str(draws(42, i)) for i in range(4)}) == 4 + assert draws(42, 1) != draws(43, 1) diff --git a/vllm/utils/torch_utils.py b/vllm/utils/torch_utils.py index b4a13419d5..025941eeb6 100644 --- a/vllm/utils/torch_utils.py +++ b/vllm/utils/torch_utils.py @@ -530,8 +530,12 @@ def kv_cache_dtype_str_to_dtype( return STR_DTYPE_TO_TORCH_DTYPE[kv_cache_dtype] -def set_random_seed(seed: int | None) -> None: +def set_random_seed(seed: int | None, data_parallel_index: int = 0) -> None: if seed is not None: + if data_parallel_index: + # DP engines share the seed but must not sample unseeded requests alike. + ss = np.random.SeedSequence((seed, data_parallel_index)) + seed = int(ss.generate_state(1)[0]) random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) diff --git a/vllm/v1/worker/cpu_worker.py b/vllm/v1/worker/cpu_worker.py index e36b15300b..b103215d65 100644 --- a/vllm/v1/worker/cpu_worker.py +++ b/vllm/v1/worker/cpu_worker.py @@ -268,7 +268,9 @@ class CPUWorker(Worker): self.model_runner.warming_up_model() # Reset the seed to ensure that the random state is not affected by # the model initialization and profiling. - set_random_seed(self.model_config.seed) + set_random_seed( + self.model_config.seed, self.parallel_config.data_parallel_index + ) return CompilationTimes( language_model=self.compilation_config.compilation_time, encoder=self.compilation_config.encoder_compilation_time, diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 3f416b94c7..f9b58831e4 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -1020,7 +1020,9 @@ class Worker(WorkerBase): # Reset the seed to ensure that the random state is not affected by # the model initialization and profiling. - set_random_seed(self.model_config.seed) + set_random_seed( + self.model_config.seed, self.parallel_config.data_parallel_index + ) # Eagerly trigger inductor's once-per-process lazy inits during # warmup (rather than on a later compile cache-miss at runtime).