Bump flashinfer version to 0.6.13 (#46683)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Wei Zhao
2026-06-29 09:30:57 -07:00
committed by GitHub
co-authored by Jee Jee Li
parent debec6440b
commit 4708292d48
6 changed files with 29 additions and 15 deletions
+1 -1
View File
@@ -793,7 +793,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
# Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
# https://docs.flashinfer.ai/installation.html
# From versions.json: .flashinfer.version
ARG FLASHINFER_VERSION=0.6.12
ARG FLASHINFER_VERSION=0.6.13
RUN --mount=type=cache,target=/opt/uv/cache \
uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
--index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
+2 -2
View File
@@ -257,13 +257,13 @@ RUN pip install setuptools==75.6.0 packaging==23.2 ninja==1.11.1.3 build==1.2.2.
# build flashinfer for torch nightly from source around 10 mins
# release version: v0.6.12
# release version: v0.6.13
# todo(elainewy): cache flashinfer build result for faster build
ENV CCACHE_DIR=/root/.cache/ccache
RUN --mount=type=cache,target=/root/.cache/ccache \
--mount=type=cache,target=/root/.cache/uv \
echo "git clone flashinfer..." \
&& git clone --depth 1 --branch v0.6.12 --recursive https://github.com/flashinfer-ai/flashinfer.git \
&& git clone --depth 1 --branch v0.6.13 --recursive https://github.com/flashinfer-ai/flashinfer.git \
&& cd flashinfer \
&& git submodule update --init --recursive \
&& echo "finish git clone flashinfer..." \
+1 -1
View File
@@ -68,7 +68,7 @@
"default": "true"
},
"FLASHINFER_VERSION": {
"default": "0.6.12"
"default": "0.6.13"
},
"GDRCOPY_CUDA_VERSION": {
"default": "12.8"
+2 -2
View File
@@ -10,8 +10,8 @@ torchaudio==2.11.0
torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
PyNvVideoCodec==2.0.4
# FlashInfer should be updated together with the Dockerfile
flashinfer-python==0.6.12
flashinfer-cubin==0.6.12
flashinfer-python==0.6.13
flashinfer-cubin==0.6.13
apache-tvm-ffi==0.1.9
tilelang==0.1.9
nvidia-cudnn-frontend>=1.19.1
+9 -2
View File
@@ -33,6 +33,8 @@ QUARK_MXFP4_TORCH_COMPATIBLE = find_spec("quark") is not None and (
else True
)
DEFAULT_STARTUP_MAX_WAIT_SECONDS = 1200
def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
"""Run GSM8K evaluation using our isolated script."""
@@ -127,7 +129,11 @@ def test_gsm8k_correctness(config_filename):
]
)
env_dict = eval_config.get("env", None)
startup_max_wait_seconds = eval_config.get(
"startup_max_wait_seconds", DEFAULT_STARTUP_MAX_WAIT_SECONDS
)
env_dict = dict(eval_config.get("env") or {})
env_dict["VLLM_ENGINE_READY_TIMEOUT_S"] = str(int(startup_max_wait_seconds))
print(f"Starting GSM8K evaluation for model: {eval_config['model_name']}")
print(f"Expected metric threshold: {eval_config['accuracy_threshold']}")
@@ -139,6 +145,7 @@ def test_gsm8k_correctness(config_filename):
"rocm_request_timeout_seconds", request_timeout_seconds
)
print(f"Request timeout: {request_timeout_seconds}s")
print(f"Startup max wait: {startup_max_wait_seconds}s")
print(f"Server args: {' '.join(server_args)}")
print(f"Environment variables: {env_dict}")
@@ -147,7 +154,7 @@ def test_gsm8k_correctness(config_filename):
eval_config["model_name"],
server_args,
env_dict=env_dict,
max_wait_seconds=eval_config.get("startup_max_wait_seconds", 600),
max_wait_seconds=startup_max_wait_seconds,
) as remote_server:
server_url = remote_server.url_for("v1")
print(f"Server started at: {server_url}")
+14 -7
View File
@@ -113,12 +113,6 @@ def kernel_warmup(worker: "Worker"):
)
# TODO: remove once FlashInfer upstream fixes the persistent file cache
# to resolve collisions like `use_8x4_sf_layout=True/False`, which causes
# invalid tactics to be chosen
_FLASHINFER_USE_PERSISTENT_CACHE = False
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
"""
Autotune FlashInfer operations.
@@ -135,7 +129,20 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
import vllm.utils.flashinfer as fi_utils
from vllm.distributed.parallel_state import get_world_group
if not _FLASHINFER_USE_PERSISTENT_CACHE:
use_persistent_cache = True
deepep_a2a_backends = {
"deepep_high_throughput",
"deepep_low_latency",
"deepep_v2",
}
if runner.vllm_config.parallel_config.all2all_backend in deepep_a2a_backends:
# DeepEP dispatch/combine can timeout when only rank 0
# performs autotune and falls behind other ranks.
# Thus we skip persistent cache in this case.
use_persistent_cache = False
if not use_persistent_cache:
with torch.inference_mode(), fi_utils.autotune():
runner._dummy_run(
num_tokens=runner.scheduler_config.max_num_batched_tokens,