[Attention][MLA] Per-request scheduling for MLA chunked context (#50613)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e7b8d59460
commit
b38e111d3e
@@ -11,9 +11,6 @@ from vllm._custom_ops import (
|
||||
scaled_fp8_quant,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.ops.triton_merge_attn_states import (
|
||||
mask_empty_context,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_merge_attn_states import (
|
||||
merge_attn_states as merge_attn_states_triton,
|
||||
)
|
||||
@@ -76,32 +73,6 @@ DTYPES = [torch.float32, torch.half, torch.bfloat16]
|
||||
all_case_info: list[tuple] = []
|
||||
|
||||
|
||||
def test_mask_empty_context() -> None:
|
||||
query_lens = torch.tensor([2] + [1] * 31 + [131, 1], dtype=torch.int32)
|
||||
query_start_loc = torch.cat(
|
||||
(torch.zeros(1, dtype=torch.int32), query_lens.cumsum(0))
|
||||
).cuda()
|
||||
context_lens = torch.tensor([4] * 32 + [0, 3], dtype=torch.int32)
|
||||
context_start_loc = torch.cat(
|
||||
(torch.zeros(1, dtype=torch.int32), context_lens.cumsum(0))
|
||||
).cuda()
|
||||
num_heads, num_tokens, head_dim = 4, 165, 16
|
||||
lse = torch.randn(num_heads, num_tokens, device="cuda")
|
||||
output = torch.randn(num_tokens, num_heads, head_dim, device="cuda")
|
||||
# Empty-context rows carry undefined (possibly non-finite) attention output.
|
||||
output[33:164] = float("nan")
|
||||
|
||||
expected_lse = lse.clone()
|
||||
expected_lse[:, 33:164] = float("-inf")
|
||||
expected_output = output.clone()
|
||||
expected_output[33:164] = 0.0
|
||||
|
||||
mask_empty_context(lse, output, query_start_loc, context_start_loc)
|
||||
|
||||
torch.testing.assert_close(lse, expected_lse)
|
||||
torch.testing.assert_close(output, expected_output)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("merge_fn", [merge_attn_states_cuda, merge_attn_states_triton])
|
||||
@pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16])
|
||||
def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
|
||||
@@ -117,8 +88,7 @@ def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
|
||||
)
|
||||
suffix_lse = torch.randn(num_heads, num_tokens, device="cuda")
|
||||
|
||||
# Tokens 2 and 3 are empty on both sides (mask_empty_context already zeroed
|
||||
# their outputs and set both LSEs to -inf).
|
||||
# Tokens 2 and 3 are empty on both sides.
|
||||
empty = slice(2, 4)
|
||||
prefix_lse[:, empty] = float("-inf")
|
||||
suffix_lse[:, empty] = float("-inf")
|
||||
|
||||
@@ -280,6 +280,10 @@ BATCH_SPECS = {
|
||||
"spec_decode_medium": BatchSpec(
|
||||
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[8, 8, 8, 8, 8, 8]
|
||||
),
|
||||
"chunked_context_prefill": BatchSpec(
|
||||
seq_lens=[1568, 520, 8, 80, 96, 8],
|
||||
query_lens=[32, 8, 8, 16, 16, 8],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -1049,6 +1053,7 @@ def run_attention_backend(
|
||||
k_scale: float,
|
||||
kv_cache_dtype: str = "auto",
|
||||
prefill_backend: MLAPrefillBackendEnum | None = None,
|
||||
chunked_prefill_workspace_size: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run attention computation using the specified backend's AttentionImpl."""
|
||||
|
||||
@@ -1137,6 +1142,13 @@ def run_attention_backend(
|
||||
|
||||
# Build metadata
|
||||
builder = builder_cls(kv_cache_spec, layer_names, vllm_config, device)
|
||||
if chunked_prefill_workspace_size is not None:
|
||||
builder.chunked_prefill_workspace_size = chunked_prefill_workspace_size
|
||||
builder.chunked_prefill_workspace = (
|
||||
builder.chunked_prefill_workspace.new_empty(
|
||||
chunked_prefill_workspace_size, head_size
|
||||
)
|
||||
)
|
||||
attn_metadata = builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
@@ -1156,32 +1168,7 @@ def run_attention_backend(
|
||||
return output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"batch_spec_name",
|
||||
[
|
||||
"small_decode",
|
||||
"small_prefill",
|
||||
"mixed_small",
|
||||
"medium_decode",
|
||||
"medium_prefill",
|
||||
"mixed_medium",
|
||||
"large_decode",
|
||||
"large_prefill",
|
||||
"single_decode",
|
||||
"single_prefill",
|
||||
"spec_decode_small",
|
||||
"spec_decode_medium",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["deepseek-ai/DeepSeek-R1"])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 4, 8, 16])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
|
||||
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
|
||||
@pytest.mark.parametrize(
|
||||
("prefill_backend", "qk_nope_head_dim", "v_head_dim"),
|
||||
_prefill_backend_dimension_params(),
|
||||
)
|
||||
def test_backend_correctness(
|
||||
def _run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
@@ -1194,6 +1181,7 @@ def test_backend_correctness(
|
||||
prefill_backend: MLAPrefillBackendEnum,
|
||||
qk_nope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
chunked_prefill_workspace_size: int | None = None,
|
||||
):
|
||||
"""
|
||||
Test that all backends produce similar outputs to a reference implementation
|
||||
@@ -1592,6 +1580,7 @@ def test_backend_correctness(
|
||||
q_scale=q_scale,
|
||||
k_scale=k_scale,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
chunked_prefill_workspace_size=chunked_prefill_workspace_size,
|
||||
)
|
||||
|
||||
# Use backend_idx to get the correct SDPA output for this backend
|
||||
@@ -1643,3 +1632,90 @@ def test_backend_correctness(
|
||||
summary = f"{len(failures)} backend(s) failed: {', '.join(backend_names)}"
|
||||
detailed_msg = "\n".join(failures)
|
||||
pytest.fail(f"{summary}\n{detailed_msg}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"batch_spec_name",
|
||||
[
|
||||
"small_decode",
|
||||
"small_prefill",
|
||||
"mixed_small",
|
||||
"medium_decode",
|
||||
"medium_prefill",
|
||||
"mixed_medium",
|
||||
"large_decode",
|
||||
"large_prefill",
|
||||
"single_decode",
|
||||
"single_prefill",
|
||||
"spec_decode_small",
|
||||
"spec_decode_medium",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["deepseek-ai/DeepSeek-R1"])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 4, 8, 16])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
|
||||
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
|
||||
@pytest.mark.parametrize(
|
||||
("prefill_backend", "qk_nope_head_dim", "v_head_dim"),
|
||||
_prefill_backend_dimension_params(),
|
||||
)
|
||||
def test_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
batch_spec_name: str,
|
||||
model: str,
|
||||
tensor_parallel_size: int,
|
||||
kv_cache_dtype: str,
|
||||
q_scale: float,
|
||||
k_scale: float,
|
||||
prefill_backend: MLAPrefillBackendEnum,
|
||||
qk_nope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
):
|
||||
_run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
batch_spec_name,
|
||||
model,
|
||||
tensor_parallel_size,
|
||||
kv_cache_dtype,
|
||||
q_scale,
|
||||
k_scale,
|
||||
prefill_backend,
|
||||
qk_nope_head_dim,
|
||||
v_head_dim,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prefill_backend", "qk_nope_head_dim", "v_head_dim"),
|
||||
_prefill_backend_dimension_params(),
|
||||
)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
def test_chunked_context_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
prefill_backend: MLAPrefillBackendEnum,
|
||||
qk_nope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
kv_cache_dtype: str,
|
||||
):
|
||||
"""Split, packed, and context-free requests match the SDPA reference."""
|
||||
_run_backend_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
batch_spec_name="chunked_context_prefill",
|
||||
model="deepseek-ai/DeepSeek-R1",
|
||||
tensor_parallel_size=16,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
q_scale=1.0,
|
||||
k_scale=1.0,
|
||||
prefill_backend=prefill_backend,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
chunked_prefill_workspace_size=1024,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-request scheduling of MLA chunked-context prefill.
|
||||
|
||||
These cover the metadata contract `_compute_prefill_context` relies on: every
|
||||
context row is gathered exactly once, no chunk exceeds the workspace, and a
|
||||
chunk never covers a prefill without context (which is why the partial no
|
||||
longer needs an empty-span masking pass).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
build_mla_chunked_context_metadata,
|
||||
init_mla_context_partial,
|
||||
reorg_kvcache,
|
||||
)
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
|
||||
|
||||
def build_chunked_context(
|
||||
context_lens: list[int],
|
||||
query_lens: list[int],
|
||||
workspace_size: int,
|
||||
block_size: int = BLOCK_SIZE,
|
||||
dcp_world_size: int = 1,
|
||||
dcp_local_block_size: int = 1,
|
||||
):
|
||||
query_start_loc = torch.zeros(len(query_lens) + 1, dtype=torch.int32)
|
||||
query_start_loc[1:] = torch.tensor(query_lens, dtype=torch.int32).cumsum(0)
|
||||
workspace_rows = workspace_size + workspace_size // dcp_world_size
|
||||
return build_mla_chunked_context_metadata(
|
||||
context_lens_cpu=torch.tensor(context_lens, dtype=torch.int32),
|
||||
prefill_query_start_loc_cpu=query_start_loc,
|
||||
chunked_prefill_workspace=torch.empty((workspace_rows, 1)),
|
||||
chunked_prefill_workspace_size=workspace_size,
|
||||
block_size=block_size,
|
||||
align_chunk_to_block=True,
|
||||
device=torch.device("cpu"),
|
||||
dcp_world_size=dcp_world_size,
|
||||
dcp_local_block_size=dcp_local_block_size,
|
||||
dcp_virtual_block_size=dcp_local_block_size * dcp_world_size,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"context_lens,workspace_size",
|
||||
[
|
||||
# Heterogeneous batch: the long request must not shrink the chunk the
|
||||
# short ones share.
|
||||
([960, 16, 16, 16, 16, 16, 16, 16], 1024),
|
||||
# A single request that does not fit is split on its own.
|
||||
([2048, 320], 1024),
|
||||
# Every request needs its own chunk.
|
||||
([1024, 1024, 1024], 1024),
|
||||
# Unaligned tails.
|
||||
([37, 1000, 5], 1024),
|
||||
([1], 1024),
|
||||
],
|
||||
)
|
||||
def test_chunks_gather_every_context_row_exactly_once(context_lens, workspace_size):
|
||||
"""Chunks tile the batch's context without gaps, overlap, or overflow.
|
||||
|
||||
This is the invariant the whole schedule rests on: attention over a chunk
|
||||
only sees the rows that chunk gathered, so a row counted twice or dropped
|
||||
silently corrupts the context partial.
|
||||
"""
|
||||
query_lens = [4] * len(context_lens)
|
||||
metadata = build_chunked_context(context_lens, query_lens, workspace_size)
|
||||
assert metadata is not None
|
||||
|
||||
gathered: dict[int, list[tuple[int, int]]] = {}
|
||||
previous_request_start = -1
|
||||
for chunk in metadata.chunks:
|
||||
assert chunk.num_context_tokens <= workspace_size
|
||||
assert chunk.num_context_tokens == int(chunk.cu_seq_lens[-1])
|
||||
assert chunk.token_to_seq.shape[0] == chunk.num_context_tokens
|
||||
# Chunks are emitted in request order, which is what lets the partial
|
||||
# treat only a chunk's first request as a continuation.
|
||||
assert chunk.request_slice.start >= previous_request_start
|
||||
previous_request_start = chunk.request_slice.start
|
||||
|
||||
starts = chunk.starts.tolist()
|
||||
seq_lens = chunk.seq_lens.tolist()
|
||||
assert len(starts) == len(seq_lens) == chunk.num_requests
|
||||
for offset, (start, length) in enumerate(zip(starts, seq_lens)):
|
||||
assert length > 0, "a chunk must not cover an empty context span"
|
||||
gathered.setdefault(chunk.request_slice.start + offset, []).append(
|
||||
(start, length)
|
||||
)
|
||||
|
||||
expected = {i: length for i, length in enumerate(context_lens) if length > 0}
|
||||
assert gathered.keys() == expected.keys()
|
||||
for request, spans in gathered.items():
|
||||
cursor = 0
|
||||
for start, length in spans:
|
||||
assert start == cursor
|
||||
cursor += length
|
||||
assert cursor == expected[request]
|
||||
|
||||
|
||||
def test_continuation_is_confined_to_a_chunks_first_request():
|
||||
"""Only a chunk's first request may continue an earlier chunk.
|
||||
|
||||
`accumulate_mla_context_chunk` merges the continuation token slice and
|
||||
writes the rest, so a chunk that continues a request other than its first,
|
||||
or reports the wrong boundary, folds the partial into the wrong tokens.
|
||||
"""
|
||||
context_lens = [2048, 32, 32]
|
||||
query_lens = [3, 5, 7]
|
||||
metadata = build_chunked_context(context_lens, query_lens, 1024)
|
||||
assert metadata is not None
|
||||
query_start_loc = [0, 3, 8, 15]
|
||||
|
||||
assert [chunk.is_continuation for chunk in metadata.chunks] == [
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
]
|
||||
for chunk in metadata.chunks:
|
||||
starts = chunk.starts.tolist()
|
||||
assert chunk.is_continuation == (starts[0] > 0)
|
||||
# Requests after the first always start at the beginning of their
|
||||
# context, so they can only ever be initialized, never merged.
|
||||
assert all(start == 0 for start in starts[1:])
|
||||
request_start = chunk.request_slice.start
|
||||
request_end = chunk.request_slice.stop
|
||||
assert chunk.continuation_token_end == query_start_loc[request_start + 1]
|
||||
assert chunk.token_slice == slice(
|
||||
query_start_loc[request_start], query_start_loc[request_end]
|
||||
)
|
||||
assert chunk.query_start_loc.tolist() == [
|
||||
offset - chunk.token_slice.start
|
||||
for offset in query_start_loc[request_start : request_end + 1]
|
||||
]
|
||||
|
||||
|
||||
def test_tail_splitting_minimizes_chunks():
|
||||
"""A tail request may fill one chunk and continue at the next chunk's head.
|
||||
|
||||
Without tail splitting these contexts need three chunks. Splitting on the
|
||||
block boundary reduces that to the workspace lower bound of two.
|
||||
"""
|
||||
metadata = build_chunked_context([768, 512, 512], [3, 5, 7], 1024)
|
||||
assert metadata is not None
|
||||
|
||||
assert len(metadata.chunks) == 2
|
||||
first, second = metadata.chunks
|
||||
assert first.starts.tolist() == [0, 0]
|
||||
assert first.seq_lens.tolist() == [768, 256]
|
||||
assert not first.is_continuation
|
||||
|
||||
assert second.starts.tolist() == [256, 0]
|
||||
assert second.seq_lens.tolist() == [256, 512]
|
||||
assert second.is_continuation
|
||||
|
||||
|
||||
def test_prefills_without_context_are_skipped_and_neutralized():
|
||||
"""Context-free prefills get neutral partials instead of chunk work.
|
||||
|
||||
If the full-query partial leaves either an internal or trailing gap
|
||||
uninitialized, the final merge combines the suffix with undefined scratch.
|
||||
"""
|
||||
metadata = build_chunked_context([64, 0, 64, 0], [4, 4, 4, 4], 1024)
|
||||
assert metadata is not None
|
||||
|
||||
covered = {
|
||||
request
|
||||
for chunk in metadata.chunks
|
||||
for request in range(chunk.request_slice.start, chunk.request_slice.stop)
|
||||
}
|
||||
assert covered == {0, 2}
|
||||
assert metadata.empty_token_slices == [slice(4, 8), slice(12, 16)]
|
||||
|
||||
output, output_lse = init_mla_context_partial(
|
||||
metadata,
|
||||
attn_output=torch.empty(4, 2, 3),
|
||||
attn_softmax_lse=torch.empty(2, 4),
|
||||
num_tokens=16,
|
||||
)
|
||||
assert output.shape == (16, 2, 3)
|
||||
assert output_lse.shape == (2, 16)
|
||||
for token_slice in metadata.empty_token_slices:
|
||||
assert not torch.count_nonzero(output[token_slice])
|
||||
assert torch.isneginf(output_lse[:, token_slice]).all()
|
||||
|
||||
|
||||
def test_no_context_needs_no_chunks():
|
||||
assert build_chunked_context([0, 0], [4, 4], 1024) is None
|
||||
|
||||
|
||||
def test_dcp_chunks_fit_the_per_rank_row_budget():
|
||||
"""Under DCP each rank gathers its own shard, so the budget is 1/world.
|
||||
|
||||
The local starts and lengths must tile the rank's padded context the same
|
||||
way the global ones tile the real context, since `reorg_kvcache` unpads
|
||||
using them.
|
||||
"""
|
||||
dcp_world_size, interleave, workspace_size = 2, 64, 1024
|
||||
context_lens = [3000, 200, 200]
|
||||
metadata = build_chunked_context(
|
||||
context_lens,
|
||||
[4] * len(context_lens),
|
||||
workspace_size,
|
||||
block_size=128,
|
||||
dcp_world_size=dcp_world_size,
|
||||
dcp_local_block_size=interleave,
|
||||
)
|
||||
assert metadata is not None
|
||||
virtual_block_size = interleave * dcp_world_size
|
||||
|
||||
local_cursor: dict[int, int] = {}
|
||||
for chunk in metadata.chunks:
|
||||
assert chunk.num_local_context_tokens <= workspace_size // dcp_world_size
|
||||
assert chunk.local_starts == chunk.starts.tolist()
|
||||
assert chunk.padded_local_cu_seq_lens.tolist() == [0] + list(
|
||||
torch.tensor(chunk.padded_local_seq_lens).cumsum(0)
|
||||
)
|
||||
assert (
|
||||
chunk.padded_local_token_to_seq.shape[0] == chunk.num_local_context_tokens
|
||||
)
|
||||
for offset, (start, length) in enumerate(
|
||||
zip(chunk.local_starts, chunk.padded_local_seq_lens)
|
||||
):
|
||||
request = chunk.request_slice.start + offset
|
||||
assert start == local_cursor.get(request, 0)
|
||||
local_cursor[request] = start + length
|
||||
|
||||
for request, length in enumerate(context_lens):
|
||||
padded_local = -(-length // virtual_block_size) * interleave
|
||||
assert local_cursor[request] == padded_local
|
||||
|
||||
|
||||
def test_dcp_reorg_uses_each_chunks_local_starts():
|
||||
"""Reorganization drops DCP padding at each request's own chunk offset."""
|
||||
dcp_world_size, interleave, workspace_size = 2, 64, 1024
|
||||
metadata = build_chunked_context(
|
||||
[3000, 200, 200],
|
||||
[4, 4, 4],
|
||||
workspace_size,
|
||||
block_size=128,
|
||||
dcp_world_size=dcp_world_size,
|
||||
dcp_local_block_size=interleave,
|
||||
)
|
||||
assert metadata is not None
|
||||
|
||||
for chunk in metadata.chunks:
|
||||
assert chunk.padded_local_seq_lens is not None
|
||||
assert chunk.local_context_lens_allranks is not None
|
||||
assert chunk.local_starts is not None
|
||||
|
||||
toks = chunk.num_local_context_tokens
|
||||
rank_buffers = [torch.full((toks, 1, 1), -1) for _ in range(dcp_world_size)]
|
||||
expected = []
|
||||
src_token_idx = 0
|
||||
for request, (padded_len, local_lens, local_start) in enumerate(
|
||||
zip(
|
||||
chunk.padded_local_seq_lens,
|
||||
chunk.local_context_lens_allranks,
|
||||
chunk.local_starts,
|
||||
)
|
||||
):
|
||||
for rank, local_len in enumerate(local_lens):
|
||||
actual_len = min(max(0, local_len - local_start), padded_len)
|
||||
values = (
|
||||
request * 100_000
|
||||
+ rank * 10_000
|
||||
+ torch.arange(local_start, local_start + actual_len)
|
||||
).view(-1, 1, 1)
|
||||
rank_buffers[rank][src_token_idx : src_token_idx + actual_len] = values
|
||||
expected.append(values)
|
||||
src_token_idx += padded_len
|
||||
|
||||
allgathered = torch.cat(rank_buffers)
|
||||
reorganized, _ = reorg_kvcache(
|
||||
allgathered,
|
||||
allgathered,
|
||||
padded_local_chunk_seq_lens_lst=chunk.padded_local_seq_lens,
|
||||
local_context_lens_allranks=chunk.local_context_lens_allranks,
|
||||
local_starts=chunk.local_starts,
|
||||
sum_seq_len=chunk.num_context_tokens,
|
||||
max_seq_len=chunk.max_seq_len,
|
||||
toks=toks,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(reorganized, torch.cat(expected))
|
||||
@@ -24,7 +24,7 @@ class CustomMLAPrefillBackend(MLAPrefillBackend):
|
||||
def run_prefill_new_tokens(self, q, k, v, return_softmax_lse):
|
||||
raise NotImplementedError
|
||||
|
||||
def run_prefill_context_chunk(self, chunk_idx, q, k, v):
|
||||
def run_prefill_context_chunk(self, chunk, q, k, v):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -112,7 +112,7 @@ def test_register_custom_backend_as_decorator():
|
||||
def run_prefill_new_tokens(self, q, k, v, return_softmax_lse):
|
||||
raise NotImplementedError
|
||||
|
||||
def run_prefill_context_chunk(self, chunk_idx, q, k, v):
|
||||
def run_prefill_context_chunk(self, chunk, q, k, v):
|
||||
raise NotImplementedError
|
||||
|
||||
assert MLAPrefillBackendEnum.CUSTOM.is_overridden()
|
||||
|
||||
@@ -875,7 +875,7 @@ def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expect
|
||||
_masked_mha_workspace_fits(
|
||||
batch_size=1,
|
||||
max_query_len=max_query_len,
|
||||
context_chunk_max_seq_lens=None,
|
||||
max_context_chunk_seq_len=0,
|
||||
workspace_numel=GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
|
||||
)
|
||||
is expected
|
||||
@@ -884,7 +884,7 @@ def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expect
|
||||
|
||||
def test_masked_mha_workspace_fits_accounts_for_batch_and_context():
|
||||
"""Request count and context chunk length are independent multipliers."""
|
||||
base = dict(batch_size=2, max_query_len=2048, context_chunk_max_seq_lens=[2048])
|
||||
base = dict(batch_size=2, max_query_len=2048, max_context_chunk_seq_len=2048)
|
||||
exact = math.prod(_topk_mask_shape(2, 2048, 2048))
|
||||
|
||||
assert _masked_mha_workspace_fits(**base, workspace_numel=exact)
|
||||
@@ -892,7 +892,7 @@ def test_masked_mha_workspace_fits_accounts_for_batch_and_context():
|
||||
**{**base, "batch_size": 3}, workspace_numel=exact
|
||||
)
|
||||
assert not _masked_mha_workspace_fits(
|
||||
**{**base, "context_chunk_max_seq_lens": [4096]}, workspace_numel=exact
|
||||
**{**base, "max_context_chunk_seq_len": 4096}, workspace_numel=exact
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user