[v1] torchrun compatibility (#13642)

Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
youkaichao
2025-02-23 22:47:24 +08:00
committed by GitHub
parent 9bebc9512f
commit eb24dc4a45
14 changed files with 67 additions and 24 deletions
@@ -48,6 +48,12 @@ test_consistent_across_ranks(
test_consistent_across_ranks(
llm.llm_engine.vllm_config.cache_config.num_gpu_blocks)
# make sure we can access the model parameters from the calling process
# of the `LLM` instance.
params = list(llm.llm_engine.model_executor.driver_worker.worker.model_runner.
model.parameters())
test_consistent_across_ranks(len(params))
# all ranks should have the same outputs
for output in outputs:
prompt = output.prompt
+4 -2
View File
@@ -5,6 +5,7 @@ import threading
import time
import uuid
from concurrent.futures import Future
from typing import List
import pytest
from transformers import AutoTokenizer
@@ -211,8 +212,9 @@ def test_engine_core_concurrent_batches(monkeypatch):
class DummyExecutor(UniProcExecutor):
def initialize(self, kv_cache_config: KVCacheConfig) -> None:
super().initialize(kv_cache_config)
def initialize_from_config(
self, kv_cache_configs: List[KVCacheConfig]) -> None:
super().initialize_from_config(kv_cache_configs)
# This executor actually can only run 1 batch at a time
self.semaphore = threading.Semaphore(1)