[Attention] FlashAttention 4 SM100 FP8 kv cache support (#42569)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b23bd73f54
commit
bd091079cb
@@ -33,6 +33,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
EncoderOnlyAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
|
||||
@@ -178,6 +179,7 @@ def create_standard_kv_cache_spec(
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
sliding_window=vllm_config.model_config.get_sliding_window(),
|
||||
kv_quant_mode=get_kv_quant_mode(vllm_config.cache_config.cache_dtype),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user