[Bugfix][MLA] Reserve sparse prefill buffers before KV cache sizing (#57575)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
Robert Shaw
parent
0390299309
commit
62af3df734
@@ -421,7 +421,18 @@ def test_sparse_backend_decode_correctness(
|
||||
workspace_init,
|
||||
q_scale: float,
|
||||
k_scale: float,
|
||||
monkeypatch,
|
||||
):
|
||||
if (
|
||||
batch_name == "large_q_pure_prefill"
|
||||
and backend_cls == FlashMLASparseBackend
|
||||
and kv_cache_dtype == "fp8_ds_mla"
|
||||
and tensor_parallel_size == 4
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.flashmla_sparse.split_prefill_chunks",
|
||||
lambda rows, capacity: [(i, i + 1) for i in range(len(rows))],
|
||||
)
|
||||
if kv_cache_dtype not in backend_cls.supported_kv_cache_dtypes:
|
||||
pytest.skip(f"{backend_cls.get_name()} does not support {kv_cache_dtype}")
|
||||
|
||||
@@ -774,6 +785,10 @@ def test_sparse_backend_decode_correctness(
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
if backend_cls == FlashMLASparseBackend and kv_cache_dtype == "fp8_ds_mla":
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
current_workspace_manager().lock()
|
||||
backend_output = mock_layer.forward_impl(
|
||||
query_vllm,
|
||||
kv_c_vllm,
|
||||
|
||||
Reference in New Issue
Block a user