[Model] Add support for openPangu moe model (#28775)
Signed-off-by: yuantao <2422264527@qq.com> Signed-off-by: yt0428 <51468697+yt0428@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Lucas Wilkinson
Cyrus Leung
parent
7157596103
commit
3f52fa5aa2
@@ -15,6 +15,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SinkFullAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
@@ -882,6 +883,30 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
|
||||
raise NotImplementedError("CrossAttentionManager does not support caching")
|
||||
|
||||
|
||||
class SinkFullAttentionManager(FullAttentionManager):
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: SinkFullAttentionSpec,
|
||||
block_pool: BlockPool,
|
||||
enable_caching: bool,
|
||||
kv_cache_group_id: int,
|
||||
dcp_world_size: int = 1,
|
||||
pcp_world_size: int = 1,
|
||||
):
|
||||
super().__init__(
|
||||
kv_cache_spec,
|
||||
block_pool,
|
||||
enable_caching,
|
||||
kv_cache_group_id,
|
||||
dcp_world_size,
|
||||
pcp_world_size,
|
||||
)
|
||||
sink_len = kv_cache_spec.sink_len
|
||||
assert sink_len is not None and sink_len > 0 and sink_len % self.block_size == 0
|
||||
num_sink_block = sink_len // self.block_size
|
||||
self.sink_blocks = self.block_pool.free_block_queue.popleft_n(num_sink_block)
|
||||
|
||||
|
||||
spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = {
|
||||
FullAttentionSpec: FullAttentionManager,
|
||||
MLAAttentionSpec: FullAttentionManager,
|
||||
@@ -889,6 +914,7 @@ spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = {
|
||||
ChunkedLocalAttentionSpec: ChunkedLocalAttentionManager,
|
||||
MambaSpec: MambaManager,
|
||||
CrossAttentionSpec: CrossAttentionManager,
|
||||
SinkFullAttentionSpec: SinkFullAttentionManager,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user