[bugfix]Indexer init skip and MTP TopK share for iteration (#45895)
Signed-off-by: JaredforReal <w13431838023@gmail.com>
This commit is contained in:
@@ -349,6 +349,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
attn_backend: type[AttentionBackend] | None = None,
|
||||
use_sparse: bool = False,
|
||||
indexer: object | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
**extra_impl_args,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -437,6 +438,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
cache_config.enable_prefix_caching = False
|
||||
|
||||
# Sparse MLA reads top-k indices from a shared buffer. Pass it
|
||||
# explicitly so backbone "skip" layers (indexer=None) still find it.
|
||||
if use_sparse:
|
||||
extra_impl_args["topk_indices_buffer"] = topk_indices_buffer
|
||||
|
||||
impl_cls = cast(type[MLAAttentionImpl], self.attn_backend.get_impl_cls())
|
||||
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an MLAAttentionImpl subclass
|
||||
num_heads=self.num_heads,
|
||||
|
||||
@@ -112,6 +112,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
use_sparse=self.is_sparse,
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
)
|
||||
|
||||
self.prefix = prefix
|
||||
|
||||
Reference in New Issue
Block a user