[Core] Allow full cudagraph with separate attention routines and orthogonal to compilation, add support for FA2 and FlashInfer (#20059)

Signed-off-by: fhl <2410591650@qq.com>
Signed-off-by: fhl2000 <63384265+fhl2000@users.noreply.github.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
fhl2000
2025-08-15 10:01:39 -04:00
committed by GitHub
co-authored by Luka Govedič Lucas Wilkinson Lucas Wilkinson
parent a0632a3e03
commit 74f441f4b5
34 changed files with 1839 additions and 597 deletions
+15 -4
View File
@@ -7,7 +7,7 @@ import random
import sys
from datetime import timedelta
from platform import uname
from typing import TYPE_CHECKING, NamedTuple, Optional, Union
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
import numpy as np
import torch
@@ -137,6 +137,8 @@ class Platform:
additional_env_vars: list[str] = []
_global_graph_pool: Optional[Any] = None
@property
def supported_dtypes(self) -> list[torch.dtype]:
"""Returns the supported dtypes for the current platform."""
@@ -522,6 +524,15 @@ class Platform:
" attribute.", self.device_type, key)
return None
def get_global_graph_pool(self) -> Any:
"""
Return the global graph pool for the this platform.
"""
cls = self.__class__
if cls._global_graph_pool is None:
cls._global_graph_pool = self.graph_pool_handle()
return cls._global_graph_pool
@classmethod
def get_cu_count(cls, device_id: int = 0) -> int:
"""
@@ -530,11 +541,11 @@ class Platform:
raise NotImplementedError
@classmethod
def get_piecewise_backend_cls(cls) -> str:
def get_static_graph_wrapper_cls(cls) -> str:
"""
Get piecewise backend class for piecewise graph.
Get static graph wrapper class for static graph.
"""
return "vllm.compilation.base_piecewise_backend.AbstractPiecewiseBackend" # noqa
return "vllm.compilation.base_static_graph.AbstractStaticGraphWrapper"
@classmethod
def stateless_init_device_torch_dist_pg(