Bump Flashinfer version to 0.6.16.post3 (#50892)

Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com>
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: jiahanc <173873397+jiahanc@users.noreply.github.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Wei Zhao
2026-08-09 00:54:07 -07:00
committed by GitHub
co-authored by jiahanc Kevin H. Luu
parent eb24bc38cf
commit f18e10a7e1
5 changed files with 51 additions and 58 deletions
+1 -1
View File
@@ -752,7 +752,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
# Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
# https://docs.flashinfer.ai/installation.html
# From versions.json: .flashinfer.version
ARG FLASHINFER_VERSION=0.6.15.post1
ARG FLASHINFER_VERSION=0.6.16.post3
RUN --mount=type=cache,target=/opt/uv/cache \
uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
--index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
+1 -1
View File
@@ -68,7 +68,7 @@
"default": "true"
},
"FLASHINFER_VERSION": {
"default": "0.6.15.post1"
"default": "0.6.16.post3"
},
"GDRCOPY_CUDA_VERSION": {
"default": "12.8"
+2 -2
View File
@@ -14,8 +14,8 @@ PyNvVideoCodec==2.0.4
# flashinfer-cubin is not on PyPI since 0.6.14; setup.py excludes it from
# install_requires so the published wheel does not carry an unresolvable pin
--extra-index-url https://flashinfer.ai/whl/
flashinfer-python==0.6.15.post1
flashinfer-cubin==0.6.15.post1
flashinfer-python==0.6.16.post3
flashinfer-cubin==0.6.16.post3
apache-tvm-ffi==0.1.11
tilelang==0.1.12
nvidia-cudnn-frontend>=1.19.1
+36 -51
View File
@@ -229,12 +229,19 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
Without autotuning, FlashInfer will rely on heuristics, which may
be significantly slower.
Tuning is performed only on rank 0. The resulting cache is broadcast
to every rank so all ranks dispatch the same kernel tactic.
Every rank profiles the same tactics. When distributed, per-tactic
timings are averaged over the world CPU group so all ranks select the
same tactic.
"""
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
import vllm.utils.flashinfer as fi_utils
from vllm.distributed.parallel_state import get_world_group
world = get_world_group()
is_leader = world.rank_in_group == 0
tuner = AutoTuner.get()
autotune_kwargs: dict = {}
skip_ops = _flashinfer_autotune_skip_ops(runner)
if skip_ops:
@@ -244,25 +251,6 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
)
autotune_kwargs["skip_ops"] = skip_ops
use_persistent_cache = True
# When distributed, tune on every rank so the collectives stay synchronized.
if get_world_group().world_size > 1:
use_persistent_cache = False
if not use_persistent_cache:
with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs):
runner._dummy_run(
num_tokens=runner.scheduler_config.max_num_batched_tokens,
skip_eplb=True,
is_profile=True,
)
get_world_group().barrier()
return
world = get_world_group()
is_leader = world.rank_in_group == 0
cache_path = resolve_flashinfer_autotune_file(runner)
if is_leader:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
@@ -271,43 +259,40 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
# When autotuning with number of tokens m, flashinfer will autotune
# operations for all number of tokens up to m, so we only need to
# run with the max number of tokens.
# Randomize inputs to avoid every token pick the same experts,
# which lead to some EP ranks receiving no tokens and skipping their
# MoE kernel entirely, and cause hang due to all-reduce collective
# during synchronized autotuning.
dummy_run_kwargs = dict(
num_tokens=runner.scheduler_config.max_num_batched_tokens,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
)
with torch.inference_mode():
if is_leader:
with fi_utils.autotune(
tune_mode=True, cache=str(cache_path), **autotune_kwargs
):
runner._dummy_run(**dummy_run_kwargs)
else:
runner._dummy_run(**dummy_run_kwargs)
# Broadcast autotune cache from rank 0 to all other ranks so every
# rank loads the same set of chosen tactics.
tune_results: bytes | None = None
# Read cached autotune results and broadcast to all ranks.
cached_results: bytes | None = None
if is_leader and cache_path.exists():
with open(cache_path, "rb") as f:
tune_results = f.read()
tune_results = world.broadcast_object(tune_results, src=0)
if tune_results is None:
logger.warning_once(
"No FlashInfer autotune cache entries found; "
"falling back to default tactics."
)
else:
write_flashinfer_autotune_cache(cache_path, tune_results)
cached_results = f.read()
cached_results = world.broadcast_object(cached_results, src=0)
if cached_results is not None:
write_flashinfer_autotune_cache(cache_path, cached_results)
world.barrier()
from flashinfer.autotuner import AutoTuner
tuner.load_configs(str(cache_path))
AutoTuner.get().load_configs(str(cache_path))
logger.info_once(
"FlashInfer autotune cache loaded on rank %d from %s.",
world.rank_in_group,
cache_path,
)
group = world.cpu_group if world.world_size > 1 else None
set_autotune_process_group(group)
try:
with (
torch.inference_mode(),
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
):
runner._dummy_run(**dummy_run_kwargs)
finally:
set_autotune_process_group(None)
if world.world_size > 1:
world.barrier()
if is_leader:
tuner.save_configs(str(cache_path))
+11 -3
View File
@@ -5824,7 +5824,10 @@ class GPUModelRunner(
@contextmanager
def maybe_randomize_inputs(
self, input_ids: torch.Tensor | None, inputs_embeds: torch.Tensor | None
self,
input_ids: torch.Tensor | None,
inputs_embeds: torch.Tensor | None,
randomize_inputs: bool = False,
):
"""
Randomize input_ids if VLLM_RANDOMIZE_DP_DUMMY_INPUTS is set.
@@ -5834,7 +5837,9 @@ class GPUModelRunner(
"""
dp_size = self.vllm_config.parallel_config.data_parallel_size
randomize_inputs = envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1
randomize_inputs = randomize_inputs or (
envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1
)
if not randomize_inputs:
yield
elif input_ids is not None:
@@ -5911,6 +5916,7 @@ class GPUModelRunner(
is_graph_capturing: bool = False,
num_active_loras: int = 0,
profile_seq_lens: int | None = None,
randomize_inputs: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Run a dummy forward pass to warm up/profile run or capture the
@@ -6208,7 +6214,9 @@ class GPUModelRunner(
num_tokens_across_dp[:] = num_tokens_padded
with (
self.maybe_randomize_inputs(input_ids, inputs_embeds),
self.maybe_randomize_inputs(
input_ids, inputs_embeds, randomize_inputs=randomize_inputs
),
set_forward_context(
attn_metadata,
self.vllm_config,