[MoE Refactor] Remove MoE DP chunking (#39107)
Signed-off-by: Bill Nell <bnell@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -1588,9 +1588,6 @@ class EngineArgs:
|
||||
|
||||
self._check_feature_supported()
|
||||
self._set_default_chunked_prefill_and_prefix_caching_args(model_config)
|
||||
self._set_default_max_num_seqs_and_batched_tokens_args(
|
||||
usage_context, model_config
|
||||
)
|
||||
self._set_default_reasoning_config_args()
|
||||
sliding_window: int | None = None
|
||||
if not is_interleaved(model_config.hf_text_config):
|
||||
@@ -1846,6 +1843,12 @@ class EngineArgs:
|
||||
target_parallel_config=parallel_config,
|
||||
)
|
||||
|
||||
self._set_default_max_num_seqs_and_batched_tokens_args(
|
||||
usage_context,
|
||||
model_config,
|
||||
parallel_config,
|
||||
)
|
||||
|
||||
assert self.max_num_batched_tokens is not None, (
|
||||
"max_num_batched_tokens must be set by this point"
|
||||
)
|
||||
@@ -2244,6 +2247,7 @@ class EngineArgs:
|
||||
self,
|
||||
usage_context: UsageContext | None,
|
||||
model_config: ModelConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
):
|
||||
world_size = self.pipeline_parallel_size * self.tensor_parallel_size
|
||||
(
|
||||
@@ -2255,10 +2259,15 @@ class EngineArgs:
|
||||
orig_max_num_seqs = self.max_num_seqs
|
||||
|
||||
if self.max_num_batched_tokens is None:
|
||||
self.max_num_batched_tokens = default_max_num_batched_tokens.get(
|
||||
usage_context,
|
||||
SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS,
|
||||
)
|
||||
if parallel_config.use_batched_dp_moe:
|
||||
self.max_num_batched_tokens = (
|
||||
SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
|
||||
)
|
||||
else:
|
||||
self.max_num_batched_tokens = default_max_num_batched_tokens.get(
|
||||
usage_context,
|
||||
SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS,
|
||||
)
|
||||
|
||||
if self.max_num_seqs is None:
|
||||
self.max_num_seqs = default_max_num_seqs.get(
|
||||
|
||||
Reference in New Issue
Block a user