[Core] shm tensor arena for efficient cpu->gpu worker broadcast (#51207)

Signed-off-by: Bolin Sun <bolins@nvidia.com>
Signed-off-by: <bolins@nvidia.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Bolin Sun
2026-10-09 16:13:24 -07:00
committed by GitHub
co-authored by Claude Nick Hill
parent fa94ad4143
commit 04b02fc8a5
8 changed files with 1290 additions and 8 deletions
+6
View File
@@ -622,6 +622,7 @@ class EngineArgs:
allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
enforce_eager: bool = ModelConfig.enforce_eager
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
enable_shm_tensor_arena: bool = ParallelConfig.enable_shm_tensor_arena
language_model_only: bool = MultiModalConfig.language_model_only
limit_mm_per_prompt: dict[str, int | dict[str, int]] = get_field(
MultiModalConfig, "limit_per_prompt"
@@ -1274,6 +1275,10 @@ class EngineArgs:
"--all2all-backend", **parallel_kwargs["all2all_backend"]
)
parallel_group.add_argument("--enable-dbo", **parallel_kwargs["enable_dbo"])
parallel_group.add_argument(
"--enable-shm-tensor-arena",
**parallel_kwargs["enable_shm_tensor_arena"],
)
parallel_group.add_argument(
"--ubatch-size",
**parallel_kwargs["ubatch_size"],
@@ -2534,6 +2539,7 @@ class EngineArgs:
enable_elastic_ep=self.enable_elastic_ep,
elastic_ep_max_dp_size=self.elastic_ep_max_dp_size,
enable_dbo=self.enable_dbo,
enable_shm_tensor_arena=self.enable_shm_tensor_arena,
ubatch_size=self.ubatch_size,
dbo_decode_token_threshold=self.dbo_decode_token_threshold,
dp_sync_interval=self.dp_sync_interval,