[Attention] Extract KV-cache update from CPU attention backend (#40470)
Signed-off-by: Diego Maniloff <diego.maniloff@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
Matthew Bonanni
parent
dc68bd8c41
commit
ba94a3b998
@@ -42,6 +42,8 @@ _CPU_ARCH_PREFER_MIXED_BATCH = (
|
||||
|
||||
|
||||
class CPUAttentionBackend(AttentionBackend):
|
||||
forward_includes_kv_cache_update: bool = False
|
||||
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
@@ -106,7 +108,6 @@ class CPUAttentionBackend(AttentionBackend):
|
||||
|
||||
@dataclass
|
||||
class CPUAttentionMetadata:
|
||||
isa: str
|
||||
num_actual_tokens: int # Number of tokens excluding padding.
|
||||
max_query_len: int
|
||||
query_start_loc: torch.Tensor
|
||||
@@ -217,7 +218,6 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
||||
)
|
||||
|
||||
attn_metadata = CPUAttentionMetadata(
|
||||
isa=self.isa,
|
||||
num_actual_tokens=num_actual_tokens,
|
||||
max_query_len=max_query_len,
|
||||
query_start_loc=query_start_loc,
|
||||
@@ -345,26 +345,6 @@ class CPUAttentionBackendImpl(AttentionImpl):
|
||||
kv_cache = kv_cache.view((num_blocks, num_kv_heads, block_size * 2, -1))
|
||||
key_cache, value_cache = kv_cache.chunk(2, dim=2)
|
||||
|
||||
# key and value may be None in the case of cross attention. They are
|
||||
# calculated once based on the output from the encoder and then cached
|
||||
# in KV cache.
|
||||
if (
|
||||
self.kv_sharing_target_layer_name is None
|
||||
and key is not None
|
||||
and value is not None
|
||||
):
|
||||
ops.cpu_attn_reshape_and_cache(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
attn_metadata.slot_mapping,
|
||||
attn_metadata.isa,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
if attn_metadata.use_sdpa_prefill:
|
||||
assert self.sinks is None, "Attention sink is unsupported in SDPA prefill"
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
@@ -401,6 +381,35 @@ class CPUAttentionBackendImpl(AttentionImpl):
|
||||
|
||||
return output
|
||||
|
||||
def do_kv_cache_update(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
if self.attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER):
|
||||
return
|
||||
|
||||
num_blocks, num_kv_heads, block_size, _ = kv_cache.size()
|
||||
kv_cache = kv_cache.view((num_blocks, num_kv_heads, block_size * 2, -1))
|
||||
key_cache, value_cache = kv_cache.chunk(2, dim=2)
|
||||
isa = _get_attn_isa(
|
||||
key.dtype, key_cache.shape[2], self.head_size, self.kv_cache_dtype
|
||||
)
|
||||
ops.cpu_attn_reshape_and_cache(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
slot_mapping,
|
||||
isa,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
def _run_sdpa_forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user