Better integration with Ray Serve (#1821)

Co-authored-by: FlorianJoncour <florian@zetta-sys.com>
This commit is contained in:
FlorianJoncour
2023-11-29 13:25:43 -08:00
committed by GitHub
co-authored by FlorianJoncour
parent a7b3e33078
commit 0229c386c5
2 changed files with 5 additions and 5 deletions
+2 -2
View File
@@ -10,7 +10,7 @@ try:
import ray
from ray.air.util.torch_dist import TorchDistributedWorker
class RayWorker(TorchDistributedWorker):
class RayWorkerVllm(TorchDistributedWorker):
"""Ray wrapper for vllm.worker.Worker, allowing Worker to be
lazliy initialized after Ray sets CUDA_VISIBLE_DEVICES."""
@@ -36,7 +36,7 @@ except ImportError as e:
"`pip install ray pandas pyarrow`.")
ray = None
TorchDistributedWorker = None
RayWorker = None
RayWorkerVllm = None
if TYPE_CHECKING:
from ray.util.placement_group import PlacementGroup