[Bugfix][GLM-5.3-Flash] SM90 sparse MLA: index_kpool mismatch leads to corruption via unread query token (#58704)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
Robert Shaw
parent
f339e0e757
commit
2edbb32e58
@@ -330,7 +330,7 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
|
||||
hf_config = vllm_config.model_config.hf_text_config
|
||||
assert hf_config.index_topk is not None
|
||||
self._index_topk = int(hf_config.index_topk)
|
||||
self._index_kpool = int(kv_cache_spec.tokens_per_state)
|
||||
self._index_kpool = int(getattr(hf_config, "index_kpool", 1) or 1)
|
||||
|
||||
def _kv_lens_host(self, cam: CommonAttentionMetadata) -> tuple[int, torch.Tensor]:
|
||||
"""Exact per-row KV lengths, host-side (the flashinfer wrapper bakes
|
||||
@@ -388,8 +388,7 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
|
||||
+ 1
|
||||
)
|
||||
topk = self._index_topk
|
||||
kpool = max(self._index_kpool, 1)
|
||||
lens = torch.where(ctx <= topk, ctx, topk + ctx % kpool)
|
||||
lens = torch.where(ctx <= topk, ctx, topk + ctx % self._index_kpool)
|
||||
return num_rows, lens.to(torch.int32)
|
||||
|
||||
def build(
|
||||
|
||||
Reference in New Issue
Block a user