[V0 Deprecation] Refactor kv cache from list to element (#37487)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -349,10 +349,7 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
# use a placeholder kv cache tensor during init, which will be replaced
|
||||
# by bind_kv_cache
|
||||
# this variable will not be accessed if use_direct_call is True
|
||||
self.kv_cache = [
|
||||
torch.tensor([])
|
||||
for _ in range(vllm_config.parallel_config.pipeline_parallel_size)
|
||||
]
|
||||
self.kv_cache = torch.tensor([])
|
||||
|
||||
# Initialize KV cache quantization attributes
|
||||
_init_kv_cache_quant(self, quant_config, prefix)
|
||||
@@ -599,7 +596,7 @@ def get_attention_context(
|
||||
if isinstance(attn_metadata, dict):
|
||||
attn_metadata = attn_metadata[layer_name]
|
||||
attn_layer: Attention | MLAAttention = forward_context.no_compile_layers[layer_name]
|
||||
kv_cache = attn_layer.kv_cache[0]
|
||||
kv_cache = attn_layer.kv_cache
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
assert isinstance(slot_mapping, dict), (
|
||||
f"Expected slot_mapping to be a dict, got {type(slot_mapping)}. "
|
||||
|
||||
@@ -415,12 +415,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
self.kv_cache = [
|
||||
torch.tensor([])
|
||||
for _ in range(
|
||||
get_current_vllm_config().parallel_config.pipeline_parallel_size
|
||||
)
|
||||
]
|
||||
self.kv_cache = torch.tensor([])
|
||||
|
||||
self.use_sparse = use_sparse
|
||||
|
||||
@@ -479,7 +474,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if isinstance(attn_metadata, dict):
|
||||
attn_metadata = attn_metadata[self.layer_name]
|
||||
self_kv_cache = self.kv_cache[0]
|
||||
self_kv_cache = self.kv_cache
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
|
||||
assert isinstance(slot_mapping, dict), (
|
||||
@@ -939,7 +934,7 @@ def unified_mla_kv_cache_update(
|
||||
return torch.empty(0, device=kv_c_normed.device, dtype=kv_c_normed.dtype)
|
||||
|
||||
attn_layer = forward_context.no_compile_layers[layer_name]
|
||||
kv_cache = attn_layer.kv_cache[0]
|
||||
kv_cache = attn_layer.kv_cache
|
||||
|
||||
slot_mapping = forward_context.slot_mapping
|
||||
assert isinstance(slot_mapping, dict), (
|
||||
|
||||
@@ -168,7 +168,7 @@ class StaticSinkAttention(Attention, CustomOp):
|
||||
"sink_key and sink_value have not been prepared"
|
||||
)
|
||||
if not self.sink_populated:
|
||||
self_kv_cache = self.kv_cache[0]
|
||||
self_kv_cache = self.kv_cache
|
||||
torch.ops.vllm.maybe_populate_sink(self_kv_cache, self.layer_name)
|
||||
|
||||
return super().forward(query, key, value, output_shape)
|
||||
|
||||
Reference in New Issue
Block a user