Fix num_gpus when TP > 1 (#1852)
This commit is contained in:
@@ -159,9 +159,13 @@ class LLMEngine:
|
||||
for bundle in placement_group.bundle_specs:
|
||||
if not bundle.get("GPU", 0):
|
||||
continue
|
||||
if self.parallel_config.tensor_parallel_size == 1:
|
||||
num_gpus = self.cache_config.gpu_memory_utilization
|
||||
else:
|
||||
num_gpus = 1
|
||||
worker = ray.remote(
|
||||
num_cpus=0,
|
||||
num_gpus=self.cache_config.gpu_memory_utilization,
|
||||
num_gpus=num_gpus,
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=placement_group,
|
||||
placement_group_capture_child_tasks=True),
|
||||
|
||||
Reference in New Issue
Block a user