[Elastic EP] Reuse CUDA graphs across reconfiguration (#54985)

This commit is contained in:
Itay Alroy
2026-09-12 10:26:39 -04:00
committed by GitHub
parent 13e221f830
commit 658c8131c7
22 changed files with 372 additions and 270 deletions
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
import vllm.envs as envs
if TYPE_CHECKING:
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
@@ -40,6 +41,44 @@ def resolve_flashinfer_autotune_file(runner: "GPUModelRunner") -> Path:
return output_dir / "autotune_configs.json"
def sync_flashinfer_autotune_cache(
runner: "GPUModelRunner",
group: "GroupCoordinator",
) -> None:
cache: bytes | str | None = None
if (
group.rank_in_group == 0
and runner.vllm_config.kernel_config.enable_flashinfer_autotune
):
try:
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
if has_flashinfer() and current_platform.has_device_capability(90):
from flashinfer.autotuner import AutoTuner
with tempfile.TemporaryDirectory() as temp_dir:
path = Path(temp_dir) / "autotune_configs.json"
AutoTuner.get().save_configs(str(path))
cache = path.read_bytes()
except Exception as exc:
cache = f"{type(exc).__name__}: {exc}"
cache = group.broadcast_object(cache)
if isinstance(cache, str):
raise RuntimeError(f"Failed to serialize FlashInfer autotune state: {cache}")
if cache is None or group.rank_in_group == 0:
return
from flashinfer.autotuner import AutoTuner
with tempfile.NamedTemporaryFile() as f:
f.write(cache)
f.flush()
if not AutoTuner.get().load_configs(f.name):
raise RuntimeError("FlashInfer autotune cache is incompatible")
def write_flashinfer_autotune_cache(cache_path: Path, contents: bytes) -> None:
cache_path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path = tempfile.mkstemp(