[feature] add index share feature for DSA MTP (#44420)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9c7f7741d4
commit
32f34d3935
@@ -1018,19 +1018,20 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
is_inplace_rope=self.indexer_rope_emb.enabled(),
|
||||
)
|
||||
|
||||
# Enable IndexCache for DeepSeek models to reduce redundant top-k
|
||||
# token selection computations in sparse attention.
|
||||
use_index_cache = getattr(config, "use_index_cache", False)
|
||||
if use_index_cache:
|
||||
# IndexCache config
|
||||
# Refer: https://arxiv.org/abs/2603.12201 for more details.
|
||||
_index_topk_freq = getattr(config, "index_topk_freq", 1)
|
||||
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
|
||||
layer_id = extract_layer_index(prefix)
|
||||
if _index_topk_pattern is None:
|
||||
_skip_topk = max(layer_id - 1, 0) % _index_topk_freq != 0
|
||||
elif 0 <= layer_id < len(_index_topk_pattern):
|
||||
_skip_topk = _index_topk_pattern[layer_id] == "S"
|
||||
# IndexCache config
|
||||
# Refer: https://arxiv.org/abs/2603.12201 for more details.
|
||||
_index_topk_freq = getattr(config, "index_topk_freq", 1)
|
||||
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
|
||||
_index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2)
|
||||
layer_id = extract_layer_index(prefix)
|
||||
|
||||
if _index_topk_pattern is None:
|
||||
_skip_topk = (
|
||||
max(layer_id - _index_skip_topk_offset + 1, 0) % _index_topk_freq
|
||||
!= 0
|
||||
)
|
||||
elif 0 <= layer_id < len(_index_topk_pattern):
|
||||
_skip_topk = _index_topk_pattern[layer_id] == "S"
|
||||
|
||||
else:
|
||||
self.indexer_rope_emb = None
|
||||
@@ -1252,8 +1253,8 @@ class DeepseekV2Model(nn.Module):
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: DeepseekV2DecoderLayer(
|
||||
vllm_config,
|
||||
prefix,
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
|
||||
Reference in New Issue
Block a user