[GLM5.3 Perf] Size the GLM-5 sparse indexer decode workspace, 3072 MiB GPU memory saved (#57701)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -296,7 +296,7 @@ class Indexer(nn.Module):
|
||||
cache_config=cache_config,
|
||||
index_kpool=self.index_kpool,
|
||||
)
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
self.max_pool_len = vllm_config.model_config.max_model_len // self.index_kpool
|
||||
self.prefix = prefix
|
||||
from vllm.v1.attention.backends.mla.indexer import get_max_prefill_buffer_size
|
||||
|
||||
@@ -307,7 +307,7 @@ class Indexer(nn.Module):
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_pool_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
tail_cache=self.tail_cache,
|
||||
|
||||
Reference in New Issue
Block a user