[Elastic EP] Support Model Runner V2 (#53934)
Signed-off-by: almogtavor <almogtavor@gmail.com> Signed-off-by: Almog Tavor <70065337+almogtavor@users.noreply.github.com>
This commit is contained in:
@@ -234,10 +234,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Persistent buffer for intermediate tensors (non-first PP ranks).
|
||||
self.intermediate_tensors: IntermediateTensors | None = None
|
||||
|
||||
# Data parallelism.
|
||||
self.dp_size = self.parallel_config.data_parallel_size
|
||||
self.dp_rank = self.parallel_config.data_parallel_rank
|
||||
|
||||
# Dual batch overlap. Created in initialize_kv_cache(), once everything
|
||||
# it runs the microbatched forward with exists.
|
||||
self.ubatch_runner: UBatchRunner | None = None
|
||||
@@ -509,9 +505,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.is_pooling_model and self.is_last_pp_rank:
|
||||
self.pooling_runner = PoolingRunner(self.model, self.vllm_config)
|
||||
eplb_models_added |= self.eplb.maybe_register_model(
|
||||
self.model, self.model_config, load_dummy_weights
|
||||
self.model, self.model_config
|
||||
)
|
||||
self.eplb.maybe_start_async_loop(eplb_models_added)
|
||||
self.eplb.maybe_start_async_loop(eplb_models_added, load_dummy_weights)
|
||||
|
||||
if not self.is_first_pp_rank:
|
||||
# For non-first PP ranks, create intermediate tensors sized
|
||||
@@ -526,6 +522,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
get_offloader().post_init()
|
||||
|
||||
@property
|
||||
def dp_size(self) -> int:
|
||||
# Not cached: elastic EP rewrites parallel_config in place on reconfigure.
|
||||
return self.parallel_config.data_parallel_size
|
||||
|
||||
@property
|
||||
def dp_rank(self) -> int:
|
||||
# Not cached: elastic EP rewrites parallel_config in place on reconfigure.
|
||||
return self.parallel_config.data_parallel_rank
|
||||
|
||||
def get_model(self) -> nn.Module:
|
||||
return self.model
|
||||
|
||||
@@ -2337,9 +2343,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def setup_eplb_from_mapping(
|
||||
self, expanded_physical_to_logical: torch.Tensor
|
||||
) -> None:
|
||||
self.eplb.setup_from_mapping(
|
||||
self.model, self.model_config, expanded_physical_to_logical
|
||||
)
|
||||
self.eplb.setup_from_mapping(self.model_config, expanded_physical_to_logical)
|
||||
|
||||
########### EPLB methods end ###########
|
||||
|
||||
|
||||
Reference in New Issue
Block a user