[Core][Kernels] Use FlashInfer backend for FP8 KV Cache when available. (#7798)

Co-authored-by: Simon Mo <simon.mo@hey.com>
This commit is contained in:
Pavani Majety
2024-08-28 10:01:22 -07:00
committed by GitHub
co-authored by Simon Mo
parent ef9baee3c5
commit b98cc28f91
3 changed files with 249 additions and 12 deletions
+23 -6
View File
@@ -83,6 +83,15 @@ class FlashInferBackend(AttentionBackend):
def get_supported_head_sizes() -> List[int]:
return [64, 128, 256]
@staticmethod
def get_fp8_dtype_for_flashinfer(kv_cache_dtype: str) -> torch.dtype:
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
return torch.float8_e4m3fn
elif kv_cache_dtype == "fp8_e5m2":
return torch.float8_e5m2
else:
return ValueError(f"Unrecognized FP8 dtype: {kv_cache_dtype}")
class FlashInferState(AttentionState):
@@ -177,9 +186,9 @@ class FlashInferState(AttentionState):
self._graph_decode_workspace_buffer, _indptr_buffer,
self._graph_indices_buffer, _last_page_len_buffer, "NHD",
use_tensor_cores)
kv_cache_dtype = get_kv_cache_torch_dtype(
self.runner.kv_cache_dtype, self.runner.model_config.dtype)
kv_cache_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer(
self.runner.kv_cache_dtype)
paged_kv_indptr_tensor_host = torch.arange(0,
batch_size + 1,
dtype=torch.int32)
@@ -340,7 +349,7 @@ class FlashInferMetadata(AttentionMetadata):
self.page_size,
# Disable flashinfer's pos encoding and use vllm's rope.
pos_encoding_mode="NONE",
data_type=self.data_type)
)
def asdict_zerocopy(self,
skip_fields: Optional[Set[str]] = None
@@ -366,7 +375,8 @@ class FlashInferMetadata(AttentionMetadata):
def decode_metadata(self) -> Optional["FlashInferMetadata"]:
# Currently chunked prefill is not supported
if self.num_prefills > 0:
assert self.num_decode_tokens == 0
assert self.num_decode_tokens == 0, (
"Chunked prefill is not supported with flashinfer yet.")
return None
return self
@@ -578,6 +588,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
kv_cache_dtype = get_kv_cache_torch_dtype(
self.runner.kv_cache_dtype, self.runner.model_config.dtype)
return FlashInferMetadata(
num_prefills=self.num_prefills,
slot_mapping=slot_mapping_tensor,
@@ -661,7 +672,6 @@ class FlashInferImpl(AttentionImpl):
if attn_metadata.num_decode_tokens > 0:
assert attn_metadata.num_prefill_tokens == 0, (
"Chunked prefill is not supported with flashinfer yet.")
if kv_cache is not None:
# Use the same reshape and cache kernel as flash attention.
ops.reshape_and_cache_flash(
@@ -674,6 +684,11 @@ class FlashInferImpl(AttentionImpl):
k_scale,
v_scale,
)
# The FlashInfer api requires data to be in fp8_e4m3 or fp8_e5m2
# to process the cache in fp8
torch_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer(
self.kv_cache_dtype)
kv_cache = kv_cache.view(torch_dtype)
query = query.contiguous(
) # Flashinfer requires query to be contiguous
@@ -711,5 +726,7 @@ class FlashInferImpl(AttentionImpl):
query,
kv_cache,
sm_scale=self.scale,
logits_soft_cap=self.logits_soft_cap)
logits_soft_cap=self.logits_soft_cap,
k_scale=k_scale,
v_scale=v_scale)
return output.view(num_tokens, hidden_size)
+4
View File
@@ -226,6 +226,10 @@ def which_attn_to_use(
elif kv_cache_dtype is not None and kv_cache_dtype.startswith("fp8"):
logger.info(
"Cannot use FlashAttention-2 backend for FP8 KV cache.")
logger.warning(
"Please use FlashInfer backend with FP8 KV Cache for "
"better performance by set environment "
"VLLM_ATTENTION_BACKEND=FLASHINFER")
selected_backend = _Backend.XFORMERS
elif block_size % 16 != 0:
logger.info(