Bump Flashinfer version to 0.6.16.post3 (#50892)
Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
jiahanc
Kevin H. Luu
parent
eb24bc38cf
commit
f18e10a7e1
+1
-1
@@ -752,7 +752,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
# Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
|
||||
# https://docs.flashinfer.ai/installation.html
|
||||
# From versions.json: .flashinfer.version
|
||||
ARG FLASHINFER_VERSION=0.6.15.post1
|
||||
ARG FLASHINFER_VERSION=0.6.16.post3
|
||||
RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
|
||||
--index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
|
||||
|
||||
@@ -68,7 +68,7 @@
|
||||
"default": "true"
|
||||
},
|
||||
"FLASHINFER_VERSION": {
|
||||
"default": "0.6.15.post1"
|
||||
"default": "0.6.16.post3"
|
||||
},
|
||||
"GDRCOPY_CUDA_VERSION": {
|
||||
"default": "12.8"
|
||||
|
||||
@@ -14,8 +14,8 @@ PyNvVideoCodec==2.0.4
|
||||
# flashinfer-cubin is not on PyPI since 0.6.14; setup.py excludes it from
|
||||
# install_requires so the published wheel does not carry an unresolvable pin
|
||||
--extra-index-url https://flashinfer.ai/whl/
|
||||
flashinfer-python==0.6.15.post1
|
||||
flashinfer-cubin==0.6.15.post1
|
||||
flashinfer-python==0.6.16.post3
|
||||
flashinfer-cubin==0.6.16.post3
|
||||
apache-tvm-ffi==0.1.11
|
||||
tilelang==0.1.12
|
||||
nvidia-cudnn-frontend>=1.19.1
|
||||
|
||||
@@ -229,12 +229,19 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
Without autotuning, FlashInfer will rely on heuristics, which may
|
||||
be significantly slower.
|
||||
|
||||
Tuning is performed only on rank 0. The resulting cache is broadcast
|
||||
to every rank so all ranks dispatch the same kernel tactic.
|
||||
Every rank profiles the same tactics. When distributed, per-tactic
|
||||
timings are averaged over the world CPU group so all ranks select the
|
||||
same tactic.
|
||||
"""
|
||||
from flashinfer.autotuner import AutoTuner, set_autotune_process_group
|
||||
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
tuner = AutoTuner.get()
|
||||
|
||||
autotune_kwargs: dict = {}
|
||||
skip_ops = _flashinfer_autotune_skip_ops(runner)
|
||||
if skip_ops:
|
||||
@@ -244,25 +251,6 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
)
|
||||
autotune_kwargs["skip_ops"] = skip_ops
|
||||
|
||||
use_persistent_cache = True
|
||||
|
||||
# When distributed, tune on every rank so the collectives stay synchronized.
|
||||
if get_world_group().world_size > 1:
|
||||
use_persistent_cache = False
|
||||
|
||||
if not use_persistent_cache:
|
||||
with torch.inference_mode(), fi_utils.autotune(**autotune_kwargs):
|
||||
runner._dummy_run(
|
||||
num_tokens=runner.scheduler_config.max_num_batched_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
)
|
||||
get_world_group().barrier()
|
||||
return
|
||||
|
||||
world = get_world_group()
|
||||
is_leader = world.rank_in_group == 0
|
||||
|
||||
cache_path = resolve_flashinfer_autotune_file(runner)
|
||||
if is_leader:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
@@ -271,43 +259,40 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
# When autotuning with number of tokens m, flashinfer will autotune
|
||||
# operations for all number of tokens up to m, so we only need to
|
||||
# run with the max number of tokens.
|
||||
# Randomize inputs to avoid every token pick the same experts,
|
||||
# which lead to some EP ranks receiving no tokens and skipping their
|
||||
# MoE kernel entirely, and cause hang due to all-reduce collective
|
||||
# during synchronized autotuning.
|
||||
dummy_run_kwargs = dict(
|
||||
num_tokens=runner.scheduler_config.max_num_batched_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
if is_leader:
|
||||
with fi_utils.autotune(
|
||||
tune_mode=True, cache=str(cache_path), **autotune_kwargs
|
||||
):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
else:
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
|
||||
# Broadcast autotune cache from rank 0 to all other ranks so every
|
||||
# rank loads the same set of chosen tactics.
|
||||
tune_results: bytes | None = None
|
||||
# Read cached autotune results and broadcast to all ranks.
|
||||
cached_results: bytes | None = None
|
||||
if is_leader and cache_path.exists():
|
||||
with open(cache_path, "rb") as f:
|
||||
tune_results = f.read()
|
||||
|
||||
tune_results = world.broadcast_object(tune_results, src=0)
|
||||
|
||||
if tune_results is None:
|
||||
logger.warning_once(
|
||||
"No FlashInfer autotune cache entries found; "
|
||||
"falling back to default tactics."
|
||||
)
|
||||
else:
|
||||
write_flashinfer_autotune_cache(cache_path, tune_results)
|
||||
cached_results = f.read()
|
||||
cached_results = world.broadcast_object(cached_results, src=0)
|
||||
if cached_results is not None:
|
||||
write_flashinfer_autotune_cache(cache_path, cached_results)
|
||||
world.barrier()
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
tuner.load_configs(str(cache_path))
|
||||
|
||||
AutoTuner.get().load_configs(str(cache_path))
|
||||
logger.info_once(
|
||||
"FlashInfer autotune cache loaded on rank %d from %s.",
|
||||
world.rank_in_group,
|
||||
cache_path,
|
||||
)
|
||||
group = world.cpu_group if world.world_size > 1 else None
|
||||
set_autotune_process_group(group)
|
||||
try:
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
|
||||
):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
finally:
|
||||
set_autotune_process_group(None)
|
||||
|
||||
if world.world_size > 1:
|
||||
world.barrier()
|
||||
if is_leader:
|
||||
tuner.save_configs(str(cache_path))
|
||||
|
||||
@@ -5824,7 +5824,10 @@ class GPUModelRunner(
|
||||
|
||||
@contextmanager
|
||||
def maybe_randomize_inputs(
|
||||
self, input_ids: torch.Tensor | None, inputs_embeds: torch.Tensor | None
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
inputs_embeds: torch.Tensor | None,
|
||||
randomize_inputs: bool = False,
|
||||
):
|
||||
"""
|
||||
Randomize input_ids if VLLM_RANDOMIZE_DP_DUMMY_INPUTS is set.
|
||||
@@ -5834,7 +5837,9 @@ class GPUModelRunner(
|
||||
"""
|
||||
|
||||
dp_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
randomize_inputs = envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1
|
||||
randomize_inputs = randomize_inputs or (
|
||||
envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1
|
||||
)
|
||||
if not randomize_inputs:
|
||||
yield
|
||||
elif input_ids is not None:
|
||||
@@ -5911,6 +5916,7 @@ class GPUModelRunner(
|
||||
is_graph_capturing: bool = False,
|
||||
num_active_loras: int = 0,
|
||||
profile_seq_lens: int | None = None,
|
||||
randomize_inputs: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Run a dummy forward pass to warm up/profile run or capture the
|
||||
@@ -6208,7 +6214,9 @@ class GPUModelRunner(
|
||||
num_tokens_across_dp[:] = num_tokens_padded
|
||||
|
||||
with (
|
||||
self.maybe_randomize_inputs(input_ids, inputs_embeds),
|
||||
self.maybe_randomize_inputs(
|
||||
input_ids, inputs_embeds, randomize_inputs=randomize_inputs
|
||||
),
|
||||
set_forward_context(
|
||||
attn_metadata,
|
||||
self.vllm_config,
|
||||
|
||||
Reference in New Issue
Block a user