From f18e10a7e1c1eb37e898c31989ff625d79791657 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:54:07 -0400 Subject: [PATCH] Bump Flashinfer version to 0.6.16.post3 (#50892) Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Signed-off-by: wzhao18 Co-authored-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Co-authored-by: Kevin H. Luu --- docker/Dockerfile | 2 +- docker/versions.json | 2 +- requirements/cuda.txt | 4 +- vllm/model_executor/warmup/kernel_warmup.py | 87 +++++++++------------ vllm/v1/worker/gpu_model_runner.py | 14 +++- 5 files changed, 51 insertions(+), 58 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 22af82b0b6..1679ae433f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -752,7 +752,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \ # Install FlashInfer JIT cache (requires CUDA-version-specific index URL) # https://docs.flashinfer.ai/installation.html # From versions.json: .flashinfer.version -ARG FLASHINFER_VERSION=0.6.15.post1 +ARG FLASHINFER_VERSION=0.6.16.post3 RUN --mount=type=cache,target=/opt/uv/cache \ uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \ --index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') diff --git a/docker/versions.json b/docker/versions.json index 68db26e500..ea51010c90 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -68,7 +68,7 @@ "default": "true" }, "FLASHINFER_VERSION": { - "default": "0.6.15.post1" + "default": "0.6.16.post3" }, "GDRCOPY_CUDA_VERSION": { "default": "12.8" diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 619e8d81fb..6c763d1a4c 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -14,8 +14,8 @@ PyNvVideoCodec==2.0.4 # flashinfer-cubin is not on PyPI since 0.6.14; setup.py excludes it from # install_requires so the published wheel does not carry an unresolvable pin --extra-index-url https://flashinfer.ai/whl/ -flashinfer-python==0.6.15.post1 -flashinfer-cubin==0.6.15.post1 +flashinfer-python==0.6.16.post3 +flashinfer-cubin==0.6.16.post3 apache-tvm-ffi==0.1.11 tilelang==0.1.12 nvidia-cudnn-frontend>=1.19.1 diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index 71d776d848..c35e1ac30c 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -229,12 +229,19 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: Without autotuning, FlashInfer will rely on heuristics, which may be significantly slower. - Tuning is performed only on rank 0. The resulting cache is broadcast - to every rank so all ranks dispatch the same kernel tactic. + Every rank profiles the same tactics. When distributed, per-tactic + timings are averaged over the world CPU group so all ranks select the + same tactic. """ + from flashinfer.autotuner import AutoTuner, set_autotune_process_group + import vllm.utils.flashinfer as fi_utils from vllm.distributed.parallel_state import get_world_group + world = get_world_group() + is_leader = world.rank_in_group == 0 + tuner = AutoTuner.get() + autotune_kwargs: dict = {} skip_ops = _flashinfer_autotune_skip_ops(runner) if skip_ops: @@ -244,25 +251,6 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: ) autotune_kwargs["skip_ops"] = skip_ops - use_persistent_cache = True - - # When distributed, tune on every rank so the collectives stay synchronized. - if get_world_group().world_size > 1: - use_persistent_cache = False - - if not use_persistent_cache: - with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs): - runner._dummy_run( - num_tokens=runner.scheduler_config.max_num_batched_tokens, - skip_eplb=True, - is_profile=True, - ) - get_world_group().barrier() - return - - world = get_world_group() - is_leader = world.rank_in_group == 0 - cache_path = resolve_flashinfer_autotune_file(runner) if is_leader: logger.info_once("Using FlashInfer autotune cache file: %s", cache_path) @@ -271,43 +259,40 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: # When autotuning with number of tokens m, flashinfer will autotune # operations for all number of tokens up to m, so we only need to # run with the max number of tokens. + # Randomize inputs to avoid every token pick the same experts, + # which lead to some EP ranks receiving no tokens and skipping their + # MoE kernel entirely, and cause hang due to all-reduce collective + # during synchronized autotuning. dummy_run_kwargs = dict( num_tokens=runner.scheduler_config.max_num_batched_tokens, skip_eplb=True, is_profile=True, + randomize_inputs=True, ) - with torch.inference_mode(): - if is_leader: - with fi_utils.autotune( - tune_mode=True, cache=str(cache_path), **autotune_kwargs - ): - runner._dummy_run(**dummy_run_kwargs) - else: - runner._dummy_run(**dummy_run_kwargs) - - # Broadcast autotune cache from rank 0 to all other ranks so every - # rank loads the same set of chosen tactics. - tune_results: bytes | None = None + # Read cached autotune results and broadcast to all ranks. + cached_results: bytes | None = None if is_leader and cache_path.exists(): with open(cache_path, "rb") as f: - tune_results = f.read() - - tune_results = world.broadcast_object(tune_results, src=0) - - if tune_results is None: - logger.warning_once( - "No FlashInfer autotune cache entries found; " - "falling back to default tactics." - ) - else: - write_flashinfer_autotune_cache(cache_path, tune_results) + cached_results = f.read() + cached_results = world.broadcast_object(cached_results, src=0) + if cached_results is not None: + write_flashinfer_autotune_cache(cache_path, cached_results) world.barrier() - from flashinfer.autotuner import AutoTuner + tuner.load_configs(str(cache_path)) - AutoTuner.get().load_configs(str(cache_path)) - logger.info_once( - "FlashInfer autotune cache loaded on rank %d from %s.", - world.rank_in_group, - cache_path, - ) + group = world.cpu_group if world.world_size > 1 else None + set_autotune_process_group(group) + try: + with ( + torch.inference_mode(), + fi_utils.autotune(tune_mode=True, **autotune_kwargs), + ): + runner._dummy_run(**dummy_run_kwargs) + finally: + set_autotune_process_group(None) + + if world.world_size > 1: + world.barrier() + if is_leader: + tuner.save_configs(str(cache_path)) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 6d637b326d..43f5c45323 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -5824,7 +5824,10 @@ class GPUModelRunner( @contextmanager def maybe_randomize_inputs( - self, input_ids: torch.Tensor | None, inputs_embeds: torch.Tensor | None + self, + input_ids: torch.Tensor | None, + inputs_embeds: torch.Tensor | None, + randomize_inputs: bool = False, ): """ Randomize input_ids if VLLM_RANDOMIZE_DP_DUMMY_INPUTS is set. @@ -5834,7 +5837,9 @@ class GPUModelRunner( """ dp_size = self.vllm_config.parallel_config.data_parallel_size - randomize_inputs = envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1 + randomize_inputs = randomize_inputs or ( + envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1 + ) if not randomize_inputs: yield elif input_ids is not None: @@ -5911,6 +5916,7 @@ class GPUModelRunner( is_graph_capturing: bool = False, num_active_loras: int = 0, profile_seq_lens: int | None = None, + randomize_inputs: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: """ Run a dummy forward pass to warm up/profile run or capture the @@ -6208,7 +6214,9 @@ class GPUModelRunner( num_tokens_across_dp[:] = num_tokens_padded with ( - self.maybe_randomize_inputs(input_ids, inputs_embeds), + self.maybe_randomize_inputs( + input_ids, inputs_embeds, randomize_inputs=randomize_inputs + ), set_forward_context( attn_metadata, self.vllm_config,