Bump flashinfer version to 0.6.13 (#46683)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
+1
-1
@@ -793,7 +793,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
# Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
|
||||
# https://docs.flashinfer.ai/installation.html
|
||||
# From versions.json: .flashinfer.version
|
||||
ARG FLASHINFER_VERSION=0.6.12
|
||||
ARG FLASHINFER_VERSION=0.6.13
|
||||
RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
|
||||
--index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
|
||||
|
||||
@@ -257,13 +257,13 @@ RUN pip install setuptools==75.6.0 packaging==23.2 ninja==1.11.1.3 build==1.2.2.
|
||||
|
||||
|
||||
# build flashinfer for torch nightly from source around 10 mins
|
||||
# release version: v0.6.12
|
||||
# release version: v0.6.13
|
||||
# todo(elainewy): cache flashinfer build result for faster build
|
||||
ENV CCACHE_DIR=/root/.cache/ccache
|
||||
RUN --mount=type=cache,target=/root/.cache/ccache \
|
||||
--mount=type=cache,target=/root/.cache/uv \
|
||||
echo "git clone flashinfer..." \
|
||||
&& git clone --depth 1 --branch v0.6.12 --recursive https://github.com/flashinfer-ai/flashinfer.git \
|
||||
&& git clone --depth 1 --branch v0.6.13 --recursive https://github.com/flashinfer-ai/flashinfer.git \
|
||||
&& cd flashinfer \
|
||||
&& git submodule update --init --recursive \
|
||||
&& echo "finish git clone flashinfer..." \
|
||||
|
||||
@@ -68,7 +68,7 @@
|
||||
"default": "true"
|
||||
},
|
||||
"FLASHINFER_VERSION": {
|
||||
"default": "0.6.12"
|
||||
"default": "0.6.13"
|
||||
},
|
||||
"GDRCOPY_CUDA_VERSION": {
|
||||
"default": "12.8"
|
||||
|
||||
@@ -10,8 +10,8 @@ torchaudio==2.11.0
|
||||
torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
|
||||
PyNvVideoCodec==2.0.4
|
||||
# FlashInfer should be updated together with the Dockerfile
|
||||
flashinfer-python==0.6.12
|
||||
flashinfer-cubin==0.6.12
|
||||
flashinfer-python==0.6.13
|
||||
flashinfer-cubin==0.6.13
|
||||
apache-tvm-ffi==0.1.9
|
||||
tilelang==0.1.9
|
||||
nvidia-cudnn-frontend>=1.19.1
|
||||
|
||||
@@ -33,6 +33,8 @@ QUARK_MXFP4_TORCH_COMPATIBLE = find_spec("quark") is not None and (
|
||||
else True
|
||||
)
|
||||
|
||||
DEFAULT_STARTUP_MAX_WAIT_SECONDS = 1200
|
||||
|
||||
|
||||
def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
|
||||
"""Run GSM8K evaluation using our isolated script."""
|
||||
@@ -127,7 +129,11 @@ def test_gsm8k_correctness(config_filename):
|
||||
]
|
||||
)
|
||||
|
||||
env_dict = eval_config.get("env", None)
|
||||
startup_max_wait_seconds = eval_config.get(
|
||||
"startup_max_wait_seconds", DEFAULT_STARTUP_MAX_WAIT_SECONDS
|
||||
)
|
||||
env_dict = dict(eval_config.get("env") or {})
|
||||
env_dict["VLLM_ENGINE_READY_TIMEOUT_S"] = str(int(startup_max_wait_seconds))
|
||||
|
||||
print(f"Starting GSM8K evaluation for model: {eval_config['model_name']}")
|
||||
print(f"Expected metric threshold: {eval_config['accuracy_threshold']}")
|
||||
@@ -139,6 +145,7 @@ def test_gsm8k_correctness(config_filename):
|
||||
"rocm_request_timeout_seconds", request_timeout_seconds
|
||||
)
|
||||
print(f"Request timeout: {request_timeout_seconds}s")
|
||||
print(f"Startup max wait: {startup_max_wait_seconds}s")
|
||||
print(f"Server args: {' '.join(server_args)}")
|
||||
print(f"Environment variables: {env_dict}")
|
||||
|
||||
@@ -147,7 +154,7 @@ def test_gsm8k_correctness(config_filename):
|
||||
eval_config["model_name"],
|
||||
server_args,
|
||||
env_dict=env_dict,
|
||||
max_wait_seconds=eval_config.get("startup_max_wait_seconds", 600),
|
||||
max_wait_seconds=startup_max_wait_seconds,
|
||||
) as remote_server:
|
||||
server_url = remote_server.url_for("v1")
|
||||
print(f"Server started at: {server_url}")
|
||||
|
||||
@@ -113,12 +113,6 @@ def kernel_warmup(worker: "Worker"):
|
||||
)
|
||||
|
||||
|
||||
# TODO: remove once FlashInfer upstream fixes the persistent file cache
|
||||
# to resolve collisions like `use_8x4_sf_layout=True/False`, which causes
|
||||
# invalid tactics to be chosen
|
||||
_FLASHINFER_USE_PERSISTENT_CACHE = False
|
||||
|
||||
|
||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
"""
|
||||
Autotune FlashInfer operations.
|
||||
@@ -135,7 +129,20 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
if not _FLASHINFER_USE_PERSISTENT_CACHE:
|
||||
use_persistent_cache = True
|
||||
|
||||
deepep_a2a_backends = {
|
||||
"deepep_high_throughput",
|
||||
"deepep_low_latency",
|
||||
"deepep_v2",
|
||||
}
|
||||
if runner.vllm_config.parallel_config.all2all_backend in deepep_a2a_backends:
|
||||
# DeepEP dispatch/combine can timeout when only rank 0
|
||||
# performs autotune and falls behind other ranks.
|
||||
# Thus we skip persistent cache in this case.
|
||||
use_persistent_cache = False
|
||||
|
||||
if not use_persistent_cache:
|
||||
with torch.inference_mode(), fi_utils.autotune():
|
||||
runner._dummy_run(
|
||||
num_tokens=runner.scheduler_config.max_num_batched_tokens,
|
||||
|
||||
Reference in New Issue
Block a user