[Misc] Add get_name method to attention backends (#4685)

This commit is contained in:
Woosuk Kwon
2024-05-08 09:59:31 -07:00
committed by GitHub
parent 0f9a6e3d22
commit 5510cf0e8a
7 changed files with 30 additions and 12 deletions
+2 -3
View File
@@ -9,7 +9,6 @@ import torch.nn as nn
from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
get_attn_backend)
from vllm.attention.backends.flashinfer import FlashInferBackend
from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
ParallelConfig, SchedulerConfig, VisionLanguageConfig)
from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
@@ -395,7 +394,7 @@ class ModelRunner:
dtype=seq_start_loc.dtype,
out=seq_start_loc[1:])
if self.attn_backend is FlashInferBackend:
if self.attn_backend.get_name() == "flashinfer":
attn_metadata = self.attn_backend.make_metadata(
is_prompt=True,
use_cuda_graph=False,
@@ -556,7 +555,7 @@ class ModelRunner:
device=self.device,
)
if self.attn_backend is FlashInferBackend:
if self.attn_backend.get_name() == "flashinfer":
if not hasattr(self, "flashinfer_workspace_buffer"):
# Allocate 16MB workspace buffer
# Follow the example of flashinfer: https://docs.flashinfer.ai/api/python/decode.html