[V0 Deprecation] Refactor kv cache from list to element (#37487)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -181,12 +181,10 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
# Only process layers that have kv_cache
|
||||
# attribute (attention layers) Skip non-attention
|
||||
# layers like FusedMoE/MLP etc.
|
||||
kv_cache_attr = getattr(layer, "kv_cache", None)
|
||||
if kv_cache_attr is None:
|
||||
kv_cache_layer = getattr(layer, "kv_cache", None)
|
||||
if kv_cache_layer is None:
|
||||
continue
|
||||
|
||||
kv_cache_layer = kv_cache_attr[0]
|
||||
|
||||
filename = self._generate_filename_debug(
|
||||
layer_name, request.token_ids, request.mm_hashes
|
||||
)
|
||||
|
||||
@@ -778,7 +778,7 @@ class LMCacheConnectorV1Impl:
|
||||
continue
|
||||
|
||||
if layer_name not in self.kv_caches:
|
||||
self.kv_caches[layer_name] = attn_layer.kv_cache[0]
|
||||
self.kv_caches[layer_name] = attn_layer.kv_cache
|
||||
|
||||
####################
|
||||
# Worker side APIs
|
||||
|
||||
@@ -214,7 +214,7 @@ class P2pNcclConnector(KVConnectorBase_V1):
|
||||
if kv_cache is None:
|
||||
continue
|
||||
|
||||
layer = kv_cache[0]
|
||||
layer = kv_cache
|
||||
|
||||
kv_cache = self.p2p_nccl_engine.recv_tensor(
|
||||
request.request_id + "#" + layer_name, remote_address
|
||||
|
||||
Reference in New Issue
Block a user