[Feature] KV cache per-token-head INT8/FP8 quantization (#38378)
Signed-off-by: JartX <sagformas@epdcenter.es> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: yangyang4991 <yangyang4991@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
co-authored by
yangyang4991
Wentao Ye
Isotr0py
parent
4eefbf9609
commit
2ce3d0ce36
@@ -38,6 +38,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheSpec,
|
||||
SlidingWindowSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -381,8 +382,10 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
|
||||
# for attn backends supporting query quantization
|
||||
self.query_quant = None
|
||||
if self.impl.supports_quant_query_input and self.kv_cache_dtype.startswith(
|
||||
"fp8"
|
||||
if (
|
||||
self.impl.supports_quant_query_input
|
||||
and self.kv_cache_dtype.startswith("fp8")
|
||||
and not self.kv_cache_dtype.endswith("per_token_head")
|
||||
):
|
||||
is_per_head = (
|
||||
hasattr(self, "q_scale") and self.q_scale.numel() == self.num_kv_heads
|
||||
@@ -539,6 +542,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
# Should not be called for enc-dec or encoder-only attention.
|
||||
assert self.attn_type == AttentionType.DECODER
|
||||
quant_mode = get_kv_quant_mode(self.kv_cache_dtype)
|
||||
if self.sliding_window is not None:
|
||||
assert not vllm_config.model_config.use_mla, (
|
||||
"MLA is not supported for slidingwindow"
|
||||
@@ -548,6 +552,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
else:
|
||||
@@ -557,6 +562,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size_v,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=quant_mode,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
ChunkedLocalAttentionSpec,
|
||||
KVCacheSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
|
||||
@@ -123,5 +124,6 @@ class ChunkedLocalAttention(Attention):
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
attention_chunk_size=self.attention_chunk_size,
|
||||
)
|
||||
|
||||
@@ -18,7 +18,11 @@ from vllm.v1.attention.backend import (
|
||||
subclass_attention_backend_with_overrides,
|
||||
)
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import CrossAttentionSpec, KVCacheSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CrossAttentionSpec,
|
||||
KVCacheSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -220,4 +224,5 @@ class CrossAttention(Attention):
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
)
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheSpec,
|
||||
SinkFullAttentionSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -217,6 +218,7 @@ class StaticSinkAttention(Attention, CustomOp):
|
||||
head_size_v=self.head_size_v,
|
||||
sink_len=self.sink_len,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user