[Elastic EP] Reuse CUDA graphs across reconfiguration (#54985)

This commit is contained in:
Itay Alroy
2026-09-12 10:26:39 -04:00
committed by GitHub
parent 13e221f830
commit 658c8131c7
22 changed files with 372 additions and 270 deletions
@@ -500,7 +500,8 @@ def fi_moe_largest_bucket(moe_config: "FusedMoEConfig") -> int:
For a detailed explanation, see: `docs/serving/data_parallel_deployment.md`
"""
return max(moe_config.max_num_tokens * moe_config.dp_size, 8192)
dp_size = moe_config.elastic_ep_max_dp_size or moe_config.dp_size
return max(moe_config.max_num_tokens * dp_size, 8192)
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)