[Core] shm tensor arena for efficient cpu->gpu worker broadcast (#51207)
Signed-off-by: Bolin Sun <bolins@nvidia.com> Signed-off-by: <bolins@nvidia.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude
Nick Hill
parent
fa94ad4143
commit
04b02fc8a5
@@ -622,6 +622,7 @@ class EngineArgs:
|
||||
allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
|
||||
enforce_eager: bool = ModelConfig.enforce_eager
|
||||
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
|
||||
enable_shm_tensor_arena: bool = ParallelConfig.enable_shm_tensor_arena
|
||||
language_model_only: bool = MultiModalConfig.language_model_only
|
||||
limit_mm_per_prompt: dict[str, int | dict[str, int]] = get_field(
|
||||
MultiModalConfig, "limit_per_prompt"
|
||||
@@ -1274,6 +1275,10 @@ class EngineArgs:
|
||||
"--all2all-backend", **parallel_kwargs["all2all_backend"]
|
||||
)
|
||||
parallel_group.add_argument("--enable-dbo", **parallel_kwargs["enable_dbo"])
|
||||
parallel_group.add_argument(
|
||||
"--enable-shm-tensor-arena",
|
||||
**parallel_kwargs["enable_shm_tensor_arena"],
|
||||
)
|
||||
parallel_group.add_argument(
|
||||
"--ubatch-size",
|
||||
**parallel_kwargs["ubatch_size"],
|
||||
@@ -2534,6 +2539,7 @@ class EngineArgs:
|
||||
enable_elastic_ep=self.enable_elastic_ep,
|
||||
elastic_ep_max_dp_size=self.elastic_ep_max_dp_size,
|
||||
enable_dbo=self.enable_dbo,
|
||||
enable_shm_tensor_arena=self.enable_shm_tensor_arena,
|
||||
ubatch_size=self.ubatch_size,
|
||||
dbo_decode_token_threshold=self.dbo_decode_token_threshold,
|
||||
dp_sync_interval=self.dp_sync_interval,
|
||||
|
||||
Reference in New Issue
Block a user