[Bugfix] Prevent out-of-bounds access in FlashInfer SM90 sparse MLA mixed batches (#56969)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -277,6 +277,7 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
|
||||
attention_layer = vllm_config.compilation_config.static_forward_context[
|
||||
layer_names[0]
|
||||
]
|
||||
self._attention_layer = attention_layer
|
||||
impl = attention_layer.impl
|
||||
if not isinstance(impl, FlashInferMLASparseSM90Impl):
|
||||
raise TypeError(
|
||||
@@ -376,6 +377,13 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
|
||||
# Replan every step outside any CUDA graph capture with this step's
|
||||
# exact per-row lengths; captured runs read the refreshed buffers.
|
||||
num_rows, kv_lens = self._kv_lens_host(common_attn_metadata)
|
||||
# MHA prefills are removed from q before forward_mqa. Plan only the
|
||||
# decode prefix in that case, using the same routing decision as MLA.
|
||||
if metadata.num_prefills > 0 and self._attention_layer._use_sparse_mha(
|
||||
metadata
|
||||
):
|
||||
num_rows = metadata.num_decode_tokens
|
||||
kv_lens = kv_lens[:num_rows]
|
||||
self.state.plan(num_rows, kv_lens)
|
||||
metadata.state = self.state
|
||||
return metadata
|
||||
|
||||
Reference in New Issue
Block a user