[Core] Pipeline Parallel Support (#4412)
Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
This commit is contained in:
+50
-15
@@ -173,6 +173,7 @@ class LLMEngine:
|
||||
"rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
|
||||
"trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
|
||||
"download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
|
||||
"pipeline_parallel_size=%d, "
|
||||
"disable_custom_all_reduce=%s, quantization=%s, "
|
||||
"enforce_eager=%s, kv_cache_dtype=%s, "
|
||||
"quantization_param_path=%s, device_config=%s, "
|
||||
@@ -195,6 +196,7 @@ class LLMEngine:
|
||||
load_config.download_dir,
|
||||
load_config.load_format,
|
||||
parallel_config.tensor_parallel_size,
|
||||
parallel_config.pipeline_parallel_size,
|
||||
parallel_config.disable_custom_all_reduce,
|
||||
model_config.quantization,
|
||||
model_config.enforce_eager,
|
||||
@@ -296,7 +298,11 @@ class LLMEngine:
|
||||
# Create the scheduler.
|
||||
# NOTE: the cache_config here have been updated with the numbers of
|
||||
# GPU and CPU blocks, which are profiled in the distributed executor.
|
||||
self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
|
||||
self.scheduler = [
|
||||
Scheduler(scheduler_config, cache_config, lora_config,
|
||||
parallel_config.pipeline_parallel_size)
|
||||
for _ in range(parallel_config.pipeline_parallel_size)
|
||||
]
|
||||
|
||||
# Metric Logging.
|
||||
if self.log_stats:
|
||||
@@ -513,8 +519,16 @@ class LLMEngine:
|
||||
raise ValueError(
|
||||
"Either SamplingParams or PoolingParams must be provided.")
|
||||
|
||||
# Add the sequence group to the scheduler.
|
||||
self.scheduler.add_seq_group(seq_group)
|
||||
# Add the sequence group to the scheduler with least unfinished seqs.
|
||||
costs = [
|
||||
scheduler.get_num_unfinished_seq_groups()
|
||||
for scheduler in self.scheduler
|
||||
]
|
||||
min_cost_scheduler = self.scheduler[costs.index(min(costs))]
|
||||
min_cost_scheduler.add_seq_group(seq_group)
|
||||
|
||||
def stop_remote_worker_execution_loop(self) -> None:
|
||||
self.model_executor.stop_remote_worker_execution_loop()
|
||||
|
||||
def process_model_inputs(
|
||||
self,
|
||||
@@ -684,7 +698,8 @@ class LLMEngine:
|
||||
>>> # abort the request
|
||||
>>> engine.abort_request(request_id)
|
||||
"""
|
||||
self.scheduler.abort_seq_group(request_id)
|
||||
for scheduler in self.scheduler:
|
||||
scheduler.abort_seq_group(request_id)
|
||||
|
||||
def get_model_config(self) -> ModelConfig:
|
||||
"""Gets the model configuration."""
|
||||
@@ -696,11 +711,20 @@ class LLMEngine:
|
||||
|
||||
def get_num_unfinished_requests(self) -> int:
|
||||
"""Gets the number of unfinished requests."""
|
||||
return self.scheduler.get_num_unfinished_seq_groups()
|
||||
return sum(scheduler.get_num_unfinished_seq_groups()
|
||||
for scheduler in self.scheduler)
|
||||
|
||||
def has_unfinished_requests(self) -> bool:
|
||||
"""Returns True if there are unfinished requests."""
|
||||
return self.scheduler.has_unfinished_seqs()
|
||||
return any(scheduler.has_unfinished_seqs()
|
||||
for scheduler in self.scheduler)
|
||||
|
||||
def has_unfinished_requests_for_virtual_engine(
|
||||
self, virtual_engine: int) -> bool:
|
||||
"""
|
||||
Returns True if there are unfinished requests for the virtual engine.
|
||||
"""
|
||||
return self.scheduler[virtual_engine].has_unfinished_seqs()
|
||||
|
||||
def _process_sequence_group_outputs(
|
||||
self,
|
||||
@@ -749,7 +773,8 @@ class LLMEngine:
|
||||
self.output_processor.process_outputs(seq_group, outputs)
|
||||
|
||||
# Free the finished sequence groups.
|
||||
self.scheduler.free_finished_seq_groups()
|
||||
for scheduler in self.scheduler:
|
||||
scheduler.free_finished_seq_groups()
|
||||
|
||||
# Create the outputs.
|
||||
request_outputs: List[Union[RequestOutput,
|
||||
@@ -815,7 +840,12 @@ class LLMEngine:
|
||||
>>> if not (engine.has_unfinished_requests() or example_inputs):
|
||||
>>> break
|
||||
"""
|
||||
seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
|
||||
if self.parallel_config.pipeline_parallel_size > 1:
|
||||
raise NotImplementedError(
|
||||
"Pipeline parallelism is only supported through AsyncLLMEngine "
|
||||
"as performance will be severely degraded otherwise.")
|
||||
seq_group_metadata_list, scheduler_outputs = self.scheduler[
|
||||
0].schedule()
|
||||
|
||||
if not scheduler_outputs.is_empty():
|
||||
execute_model_req = ExecuteModelRequest(
|
||||
@@ -886,23 +916,28 @@ class LLMEngine:
|
||||
|
||||
# System State
|
||||
# Scheduler State
|
||||
num_running_sys = len(self.scheduler.running)
|
||||
num_swapped_sys = len(self.scheduler.swapped)
|
||||
num_waiting_sys = len(self.scheduler.waiting)
|
||||
num_running_sys = sum(
|
||||
len(scheduler.running) for scheduler in self.scheduler)
|
||||
num_swapped_sys = sum(
|
||||
len(scheduler.swapped) for scheduler in self.scheduler)
|
||||
num_waiting_sys = sum(
|
||||
len(scheduler.waiting) for scheduler in self.scheduler)
|
||||
|
||||
# KV Cache Usage in %
|
||||
num_total_gpu = self.cache_config.num_gpu_blocks
|
||||
gpu_cache_usage_sys = 0.
|
||||
if num_total_gpu is not None:
|
||||
num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks(
|
||||
)
|
||||
num_free_gpu = sum(
|
||||
scheduler.block_manager.get_num_free_gpu_blocks()
|
||||
for scheduler in self.scheduler)
|
||||
gpu_cache_usage_sys = 1.0 - (num_free_gpu / num_total_gpu)
|
||||
|
||||
num_total_cpu = self.cache_config.num_cpu_blocks
|
||||
cpu_cache_usage_sys = 0.
|
||||
if num_total_cpu is not None and num_total_cpu > 0:
|
||||
num_free_cpu = self.scheduler.block_manager.get_num_free_cpu_blocks(
|
||||
)
|
||||
num_free_cpu = sum(
|
||||
scheduler.block_manager.get_num_free_cpu_blocks()
|
||||
for scheduler in self.scheduler)
|
||||
cpu_cache_usage_sys = 1.0 - (num_free_cpu / num_total_cpu)
|
||||
|
||||
# Iteration stats
|
||||
|
||||
Reference in New Issue
Block a user