Bump flashinfer version to 0.6.13 (#46683)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -113,12 +113,6 @@ def kernel_warmup(worker: "Worker"):
|
||||
)
|
||||
|
||||
|
||||
# TODO: remove once FlashInfer upstream fixes the persistent file cache
|
||||
# to resolve collisions like `use_8x4_sf_layout=True/False`, which causes
|
||||
# invalid tactics to be chosen
|
||||
_FLASHINFER_USE_PERSISTENT_CACHE = False
|
||||
|
||||
|
||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
"""
|
||||
Autotune FlashInfer operations.
|
||||
@@ -135,7 +129,20 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
if not _FLASHINFER_USE_PERSISTENT_CACHE:
|
||||
use_persistent_cache = True
|
||||
|
||||
deepep_a2a_backends = {
|
||||
"deepep_high_throughput",
|
||||
"deepep_low_latency",
|
||||
"deepep_v2",
|
||||
}
|
||||
if runner.vllm_config.parallel_config.all2all_backend in deepep_a2a_backends:
|
||||
# DeepEP dispatch/combine can timeout when only rank 0
|
||||
# performs autotune and falls behind other ranks.
|
||||
# Thus we skip persistent cache in this case.
|
||||
use_persistent_cache = False
|
||||
|
||||
if not use_persistent_cache:
|
||||
with torch.inference_mode(), fi_utils.autotune():
|
||||
runner._dummy_run(
|
||||
num_tokens=runner.scheduler_config.max_num_batched_tokens,
|
||||
|
||||
Reference in New Issue
Block a user