[Attention][MLA] Per-request scheduling for MLA chunked context (#50613)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e7b8d59460
commit
b38e111d3e
@@ -133,9 +133,19 @@ fixed workspace size.
|
||||
|
||||
The chunked prefill approach is as follows:
|
||||
|
||||
MCC Max chunk of context to process per iter, computed dynamically,
|
||||
W Workspace rows, i.e. the context rows we may gather at once,
|
||||
used to bound the memory usage
|
||||
|
||||
The context is scheduled per request: requests are packed in order, splitting
|
||||
the next request on an aligned boundary when needed to fill `W`.
|
||||
So a chunk covers a contiguous run of prefills and only ever the tokens and
|
||||
context rows of those prefills — attention, up-projection and merging are
|
||||
charged only to the requests it covers, and no chunk contains an empty context
|
||||
span. See `plan_mla_context_chunks`.
|
||||
|
||||
Only a chunk's first request can be a continuation, so accumulation performs at
|
||||
most one request-slice merge and one bulk write per chunk.
|
||||
|
||||
q_c = h_t @ W_DQ
|
||||
q_nope = (q_c @ W_UQ).view(Sq, N, P)
|
||||
q_pe = RoPE(q_c @ W_QR).view(Sq, N, R)
|
||||
@@ -157,10 +167,12 @@ curr_o, curr_lse = scaled_dot_product_attention(
|
||||
return_softmax_lse=True
|
||||
)
|
||||
|
||||
// Compute attention with the already existing context
|
||||
for chunk_idx in range(cdiv(C, MCC)):
|
||||
chunk_start = chunk_idx * MCC
|
||||
chunk_end = min(chunk_start + MCC, C)
|
||||
// Compute attention with the already existing context. Shown for a single
|
||||
// request; with a batch, a chunk covers a run of requests and both `q` and the
|
||||
// gathered context below are sliced to that run.
|
||||
for chunk_idx in range(cdiv(C, W)):
|
||||
chunk_start = chunk_idx * W
|
||||
chunk_end = min(chunk_start + W, C)
|
||||
Sc = chunk_end - chunk_start
|
||||
cache_kv_c_chunk = cache_kv_c[chunk_start:chunk_end]
|
||||
cache_k_pe_chunk = cache_k_pe[chunk_start:chunk_end]
|
||||
@@ -188,11 +200,15 @@ return curr_o @ W_O
|
||||
"""
|
||||
|
||||
import functools
|
||||
import itertools
|
||||
import math
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import ClassVar, Generic, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from tqdm import tqdm
|
||||
@@ -256,6 +272,7 @@ from vllm.utils.torch_utils import (
|
||||
direct_register_custom_op,
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
np_to_pinned_tensor,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
@@ -278,7 +295,6 @@ from vllm.v1.attention.backends.utils import (
|
||||
from vllm.v1.attention.ops.common import cp_lse_ag_out_ar, cp_lse_ag_out_rs
|
||||
from vllm.v1.attention.ops.dcp_alltoall import dcp_a2a_lse_reduce
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.attention.ops.triton_merge_attn_states import mask_empty_context
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
@@ -1375,28 +1391,42 @@ class MLACommonPrefillMetadata:
|
||||
"""Prefill Specific Metadata"""
|
||||
|
||||
@dataclass
|
||||
class ChunkedContextMetadata:
|
||||
# New for MLA (compared to FlashAttention)
|
||||
# For handling chunked prefill
|
||||
class ContextChunk:
|
||||
"""One workspace-sized slice of paged context for a run of prefills."""
|
||||
|
||||
index: int
|
||||
request_slice: slice
|
||||
token_slice: slice
|
||||
continuation_token_end: int
|
||||
is_continuation: bool
|
||||
num_context_tokens: int
|
||||
query_start_loc: torch.Tensor
|
||||
max_query_len: int
|
||||
cu_seq_lens: torch.Tensor
|
||||
starts: torch.Tensor
|
||||
seq_tot: list[int]
|
||||
max_seq_lens: list[int]
|
||||
max_seq_len: int
|
||||
seq_lens: torch.Tensor
|
||||
context_lens: torch.Tensor
|
||||
workspace: torch.Tensor
|
||||
token_to_seq: torch.Tensor
|
||||
chunk_total_token: list[int]
|
||||
has_empty_context: list[bool]
|
||||
|
||||
# for mla DCP
|
||||
padded_local_chunk_seq_lens: list[list[int]] | None = None
|
||||
padded_local_seq_lens: list[int] | None = None
|
||||
local_context_lens_allranks: list[list[int]] | None = None
|
||||
padded_local_cu_seq_lens: torch.Tensor | None = None
|
||||
padded_local_token_to_seq: torch.Tensor | None = None
|
||||
cu_seq_lens_lst: list[list[int]] | None = None
|
||||
chunk_size: int | None = None
|
||||
prefill_tokens_with_context: int | None = None
|
||||
num_local_context_tokens: int = 0
|
||||
local_starts: list[int] | None = None
|
||||
|
||||
@property
|
||||
def num_requests(self) -> int:
|
||||
return self.request_slice.stop - self.request_slice.start
|
||||
|
||||
@dataclass
|
||||
class ChunkedContextMetadata:
|
||||
context_lens: torch.Tensor
|
||||
workspace: torch.Tensor
|
||||
chunks: "list[MLACommonPrefillMetadata.ContextChunk]"
|
||||
context_lens_list: list[int]
|
||||
empty_token_slices: list[slice]
|
||||
|
||||
block_table: torch.Tensor
|
||||
query_start_loc: torch.Tensor
|
||||
@@ -1566,11 +1596,98 @@ def backend_supports_prefill_query_quantization() -> bool:
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ContextChunkPlan:
|
||||
"""Request-space layout of one context chunk, before tensors are built."""
|
||||
|
||||
request_start: int
|
||||
request_end: int
|
||||
# Per-request context offset and row count, in request order.
|
||||
starts: list[int]
|
||||
seq_lens: list[int]
|
||||
is_continuation: bool
|
||||
|
||||
|
||||
def plan_mla_context_chunks(
|
||||
context_lens: list[int],
|
||||
row_budget: int,
|
||||
max_context_chunk: int,
|
||||
split_alignment: int,
|
||||
padded_rows: Callable[[int], int],
|
||||
) -> list[_ContextChunkPlan]:
|
||||
"""Pack per-request contexts into workspace-sized chunks."""
|
||||
assert max_context_chunk > 0
|
||||
assert split_alignment > 0
|
||||
|
||||
def aligned_split_len(start: int, remaining: int, available: int) -> int:
|
||||
max_split = min(
|
||||
max_context_chunk,
|
||||
round_down(remaining - 1, split_alignment),
|
||||
)
|
||||
low, high = 0, max_split // split_alignment
|
||||
while low < high:
|
||||
mid = (low + high + 1) // 2
|
||||
length = mid * split_alignment
|
||||
rows = padded_rows(start + length) - padded_rows(start)
|
||||
if rows <= available:
|
||||
low = mid
|
||||
else:
|
||||
high = mid - 1
|
||||
return low * split_alignment
|
||||
|
||||
plans: list[_ContextChunkPlan] = []
|
||||
num_requests = len(context_lens)
|
||||
request = 0
|
||||
start = 0
|
||||
while request < num_requests:
|
||||
context_len = context_lens[request]
|
||||
if context_len == 0:
|
||||
assert start == 0
|
||||
request += 1
|
||||
continue
|
||||
|
||||
request_start = request
|
||||
starts: list[int] = []
|
||||
seq_lens: list[int] = []
|
||||
rows = 0
|
||||
while request < num_requests and context_lens[request] > 0:
|
||||
remaining = context_lens[request] - start
|
||||
request_rows = padded_rows(start + remaining) - padded_rows(start)
|
||||
if rows + request_rows > row_budget:
|
||||
split_len = aligned_split_len(start, remaining, row_budget - rows)
|
||||
if split_len > 0:
|
||||
starts.append(start)
|
||||
seq_lens.append(split_len)
|
||||
rows += padded_rows(start + split_len) - padded_rows(start)
|
||||
start += split_len
|
||||
break
|
||||
rows += request_rows
|
||||
starts.append(start)
|
||||
seq_lens.append(remaining)
|
||||
request += 1
|
||||
start = 0
|
||||
assert seq_lens
|
||||
plans.append(
|
||||
_ContextChunkPlan(
|
||||
request_start=request_start,
|
||||
request_end=request_start + len(seq_lens),
|
||||
starts=starts,
|
||||
seq_lens=seq_lens,
|
||||
is_continuation=starts[0] > 0,
|
||||
)
|
||||
)
|
||||
return plans
|
||||
|
||||
|
||||
def _flat_int32(values: list[int] | np.ndarray) -> torch.Tensor:
|
||||
"""Pinned int32 CPU tensor backing one concatenated per-chunk field."""
|
||||
return np_to_pinned_tensor(np.asarray(values, dtype=np.int32))
|
||||
|
||||
|
||||
def build_mla_chunked_context_metadata(
|
||||
*,
|
||||
context_lens_cpu: torch.Tensor,
|
||||
prefill_query_start_loc_cpu: torch.Tensor,
|
||||
num_prefills: int,
|
||||
chunked_prefill_workspace: torch.Tensor,
|
||||
chunked_prefill_workspace_size: int,
|
||||
block_size: int,
|
||||
@@ -1582,14 +1699,13 @@ def build_mla_chunked_context_metadata(
|
||||
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
|
||||
"""Build chunked-context metadata for an MLA prefill.
|
||||
|
||||
Shared by dense and sparse builders. Splits each prefill's context
|
||||
into workspace-sized chunks and, under DCP, plans the per-rank interleaved
|
||||
local chunks the all-gather reduction consumes.
|
||||
Shared by dense and sparse builders. Packs the prefill contexts into a flat
|
||||
list of workspace-sized per-request chunks and, under DCP, plans the
|
||||
per-rank interleaved local chunks the all-gather reduction consumes.
|
||||
|
||||
Args:
|
||||
context_lens_cpu: Per-prefill context length (seq_len - query_len).
|
||||
prefill_query_start_loc_cpu: Prefill query cumulative offsets (0-based).
|
||||
num_prefills: Number of prefill requests.
|
||||
chunked_prefill_workspace: Scratch buffer the context gather writes to.
|
||||
chunked_prefill_workspace_size: Row capacity of the workspace.
|
||||
block_size: KV cache page size for chunk-start alignment.
|
||||
@@ -1604,147 +1720,189 @@ def build_mla_chunked_context_metadata(
|
||||
"""
|
||||
# NOTE: it is recommended you read the `Chunked Prefill` section in the
|
||||
# comment at the top of the file before trying to understand this code.
|
||||
max_context_len = context_lens_cpu.max().item()
|
||||
if max_context_len <= 0:
|
||||
context_lens = context_lens_cpu.tolist()
|
||||
if max(context_lens, default=0) <= 0:
|
||||
return None
|
||||
num_prefills_with_context = int((context_lens_cpu > 0).sum().item())
|
||||
|
||||
# Currently we allocate an equal amount of workspace for each prefill with
|
||||
# context; we could probably use a more advanced algorithm here and allocate
|
||||
# more workspace to prefills with longer context lengths.
|
||||
max_context_chunk = chunked_prefill_workspace_size // num_prefills_with_context
|
||||
if align_chunk_to_block:
|
||||
# The `gather_and_maybe_dequant_cache` kernel cannot handle chunk
|
||||
# starts that are not aligned to block_size, so round down.
|
||||
max_context_chunk = round_down(max_context_chunk, block_size)
|
||||
assert max_context_chunk > 0
|
||||
|
||||
num_chunks = cdiv(max_context_len, max_context_chunk)
|
||||
# e.g. max_context_chunk=256, num_chunks=3, num_prefills=4 ->
|
||||
# [[0, 0, 0, 0], [256, 256, 256, 256], [512, 512, 512, 512]]
|
||||
# Note(simon): this is done on CPU because of downstream's use of `to_list`.
|
||||
chunk_starts = torch.empty(
|
||||
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
||||
).copy_(
|
||||
torch.arange(num_chunks, dtype=torch.int32)
|
||||
.multiply_(max_context_chunk)
|
||||
.unsqueeze(1)
|
||||
)
|
||||
chunk_ends = torch.min(
|
||||
context_lens_cpu.unsqueeze(0), chunk_starts + max_context_chunk
|
||||
)
|
||||
chunk_seq_lens = chunk_ends - chunk_starts
|
||||
chunk_seq_lens.clamp_(min=0)
|
||||
has_empty_context = torch.any(chunk_seq_lens == 0, dim=1).tolist()
|
||||
|
||||
cu_seq_lens_cpu = torch.zeros(
|
||||
num_chunks, num_prefills + 1, dtype=torch.int32, pin_memory=True
|
||||
)
|
||||
torch.cumsum(chunk_seq_lens, dim=1, out=cu_seq_lens_cpu[:, 1:], dtype=torch.int32)
|
||||
chunk_total_token = cu_seq_lens_cpu[:, -1]
|
||||
|
||||
max_tokens_over_chunk = chunk_total_token.max().item()
|
||||
token_to_seq_cpu = torch.zeros(
|
||||
(num_chunks, max_tokens_over_chunk), dtype=torch.int32, pin_memory=True
|
||||
)
|
||||
req_indices = torch.arange(num_prefills, dtype=torch.int32)
|
||||
for i in range(num_chunks):
|
||||
token_to_seq = torch.repeat_interleave(req_indices, chunk_seq_lens[i])
|
||||
token_to_seq_cpu[i, : token_to_seq.shape[0]] = token_to_seq
|
||||
|
||||
prefill_tokens_with_context = prefill_query_start_loc_cpu[
|
||||
num_prefills_with_context
|
||||
].item()
|
||||
|
||||
metadata_cls = MLACommonPrefillMetadata.ChunkedContextMetadata
|
||||
# The `gather_and_maybe_dequant_cache` kernel cannot handle chunk starts
|
||||
# that are not aligned to block_size, so a split request advances in
|
||||
# block-aligned steps.
|
||||
chunk_alignment = block_size if align_chunk_to_block else 1
|
||||
if dcp_world_size > 1:
|
||||
local_context_lens_allranks = get_dcp_local_seq_lens(
|
||||
context_lens_cpu, dcp_world_size, None, dcp_local_block_size
|
||||
)
|
||||
# Note(qcs): Per-rank local context lengths, padded to
|
||||
# `dcp_local_block_size`.
|
||||
padded_local_context_lens_cpu: torch.Tensor = (
|
||||
cdiv(context_lens_cpu, dcp_virtual_block_size) * dcp_local_block_size
|
||||
)
|
||||
# Note(hc): The above max_context_chunk already enforces block_size
|
||||
# alignment; DCP only requires block_size be divisible by dcp_world_size,
|
||||
# because DCP uses cp_gather_cache, which does not require chunk starts
|
||||
# aligned to block_size.
|
||||
assert max_context_chunk % dcp_world_size == 0
|
||||
padded_local_max_context_chunk = (
|
||||
cdiv(max_context_chunk, dcp_virtual_block_size) * dcp_local_block_size
|
||||
)
|
||||
local_chunk_starts = torch.empty(
|
||||
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
||||
).copy_(
|
||||
torch.arange(num_chunks, dtype=torch.int32)
|
||||
.multiply_(padded_local_max_context_chunk)
|
||||
.unsqueeze(1)
|
||||
)
|
||||
local_chunk_ends = torch.min(
|
||||
padded_local_context_lens_cpu.unsqueeze(0),
|
||||
local_chunk_starts + padded_local_max_context_chunk,
|
||||
)
|
||||
padded_local_chunk_seq_lens = local_chunk_ends - local_chunk_starts
|
||||
padded_local_chunk_seq_lens.clamp_(min=0)
|
||||
# Each rank gathers only its own shard, so a chunk's workspace cost is
|
||||
# 1/dcp_world_size of its context rows, rounded up to the interleave
|
||||
# block size. Sub-chunks are additionally aligned to the virtual block
|
||||
# size so that every rank's local start stays interleave-aligned.
|
||||
row_budget = chunked_prefill_workspace_size // dcp_world_size
|
||||
chunk_alignment = math.lcm(chunk_alignment, dcp_virtual_block_size)
|
||||
|
||||
padded_local_cu_seq_lens_cpu = torch.zeros(
|
||||
num_chunks, num_prefills + 1, dtype=torch.int32, pin_memory=True
|
||||
)
|
||||
torch.cumsum(
|
||||
padded_local_chunk_seq_lens,
|
||||
dim=1,
|
||||
out=padded_local_cu_seq_lens_cpu[:, 1:],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
max_padded_local_tokens = padded_local_cu_seq_lens_cpu[:, -1].max().item()
|
||||
padded_local_token_to_seq_cpu = torch.zeros(
|
||||
(num_chunks, max_padded_local_tokens), dtype=torch.int32
|
||||
)
|
||||
for i in range(num_chunks):
|
||||
tts = torch.repeat_interleave(req_indices, padded_local_chunk_seq_lens[i])
|
||||
padded_local_token_to_seq_cpu[i, : tts.shape[0]] = tts
|
||||
|
||||
chunked_context_metadata = metadata_cls(
|
||||
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
||||
starts=local_chunk_starts.to(device, non_blocking=True),
|
||||
seq_tot=padded_local_chunk_seq_lens.sum(dim=1).tolist(),
|
||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||
seq_lens=chunk_seq_lens,
|
||||
context_lens=context_lens_cpu.to(device, non_blocking=True),
|
||||
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
|
||||
chunk_total_token=chunk_total_token.tolist(),
|
||||
workspace=chunked_prefill_workspace,
|
||||
has_empty_context=has_empty_context,
|
||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||
padded_local_chunk_seq_lens=padded_local_chunk_seq_lens.tolist(),
|
||||
local_context_lens_allranks=local_context_lens_allranks.tolist(),
|
||||
padded_local_cu_seq_lens=padded_local_cu_seq_lens_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
padded_local_token_to_seq=padded_local_token_to_seq_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
cu_seq_lens_lst=cu_seq_lens_cpu.tolist(),
|
||||
chunk_size=padded_local_max_context_chunk,
|
||||
)
|
||||
def padded_rows(rows: int) -> int:
|
||||
return cdiv(rows, dcp_virtual_block_size) * dcp_local_block_size
|
||||
else:
|
||||
chunked_context_metadata = metadata_cls(
|
||||
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
||||
starts=chunk_starts.to(device, non_blocking=True),
|
||||
seq_tot=chunk_seq_lens.sum(dim=1).tolist(),
|
||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||
seq_lens=chunk_seq_lens,
|
||||
context_lens=context_lens_cpu.to(device, non_blocking=True),
|
||||
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
|
||||
chunk_total_token=chunk_total_token,
|
||||
workspace=chunked_prefill_workspace,
|
||||
has_empty_context=has_empty_context,
|
||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||
row_budget = chunked_prefill_workspace_size
|
||||
|
||||
def padded_rows(rows: int) -> int:
|
||||
return rows
|
||||
|
||||
max_context_chunk = round_down(chunked_prefill_workspace_size, chunk_alignment)
|
||||
assert max_context_chunk > 0, (
|
||||
f"chunked prefill workspace ({chunked_prefill_workspace_size} rows) is "
|
||||
f"smaller than the context chunk alignment ({chunk_alignment} rows)"
|
||||
)
|
||||
|
||||
plans = plan_mla_context_chunks(
|
||||
context_lens,
|
||||
row_budget,
|
||||
max_context_chunk,
|
||||
chunk_alignment,
|
||||
padded_rows,
|
||||
)
|
||||
|
||||
query_start_loc = prefill_query_start_loc_cpu.tolist()
|
||||
empty_token_slices = [
|
||||
slice(query_start_loc[i], query_start_loc[i + 1])
|
||||
for i, context_len in enumerate(context_lens)
|
||||
if context_len == 0
|
||||
]
|
||||
|
||||
use_dcp = dcp_world_size > 1
|
||||
local_context_lens_allranks = (
|
||||
get_dcp_local_seq_lens(
|
||||
context_lens_cpu, dcp_world_size, None, dcp_local_block_size
|
||||
).tolist()
|
||||
if use_dcp
|
||||
else None
|
||||
)
|
||||
|
||||
# Concatenate the per-chunk fields so each one costs a single
|
||||
# host-to-device transfer, and remember where every chunk's slice lands.
|
||||
starts_flat: list[int] = []
|
||||
seq_lens_flat: list[int] = []
|
||||
cu_seq_lens_flat: list[int] = []
|
||||
cu_seqlens_q_flat: list[int] = []
|
||||
token_to_seq_parts: list[np.ndarray] = []
|
||||
padded_local_cu_seq_lens_flat: list[int] = []
|
||||
padded_local_token_to_seq_parts: list[np.ndarray] = []
|
||||
local_starts_per_chunk: list[list[int]] = []
|
||||
local_seq_lens_per_chunk: list[list[int]] = []
|
||||
layouts: list[tuple[slice, slice, slice, slice]] = []
|
||||
|
||||
request_offset = boundary_offset = token_offset = local_token_offset = 0
|
||||
for plan in plans:
|
||||
num_requests = len(plan.seq_lens)
|
||||
num_tokens = sum(plan.seq_lens)
|
||||
num_local_tokens = num_tokens
|
||||
|
||||
seq_lens_flat.extend(plan.seq_lens)
|
||||
cu_seq_lens_flat.extend(itertools.accumulate(plan.seq_lens, initial=0))
|
||||
query_base = query_start_loc[plan.request_start]
|
||||
cu_seqlens_q_flat.extend(
|
||||
query_start_loc[request] - query_base
|
||||
for request in range(plan.request_start, plan.request_end + 1)
|
||||
)
|
||||
token_to_seq_parts.append(
|
||||
np.repeat(np.arange(num_requests, dtype=np.int32), plan.seq_lens)
|
||||
)
|
||||
|
||||
assert max(chunked_context_metadata.max_seq_lens) <= chunked_prefill_workspace_size
|
||||
return chunked_context_metadata
|
||||
if use_dcp:
|
||||
# A request's local rows are its context rows sharded across ranks
|
||||
# and rounded up to the interleave block size, so the local start of
|
||||
# a continuation is the padded row count of the context before it.
|
||||
local_starts = [padded_rows(start) for start in plan.starts]
|
||||
local_seq_lens = [
|
||||
padded_rows(start + length) - local_start
|
||||
for start, length, local_start in zip(
|
||||
plan.starts, plan.seq_lens, local_starts
|
||||
)
|
||||
]
|
||||
local_starts_per_chunk.append(local_starts)
|
||||
local_seq_lens_per_chunk.append(local_seq_lens)
|
||||
padded_local_cu_seq_lens_flat.extend(
|
||||
itertools.accumulate(local_seq_lens, initial=0)
|
||||
)
|
||||
padded_local_token_to_seq_parts.append(
|
||||
np.repeat(np.arange(num_requests, dtype=np.int32), local_seq_lens)
|
||||
)
|
||||
num_local_tokens = sum(local_seq_lens)
|
||||
# The gather takes per-rank local offsets under DCP.
|
||||
starts_flat.extend(local_starts)
|
||||
else:
|
||||
starts_flat.extend(plan.starts)
|
||||
assert num_local_tokens <= row_budget
|
||||
|
||||
layouts.append(
|
||||
(
|
||||
slice(request_offset, request_offset + num_requests),
|
||||
slice(boundary_offset, boundary_offset + num_requests + 1),
|
||||
slice(token_offset, token_offset + num_tokens),
|
||||
slice(local_token_offset, local_token_offset + num_local_tokens),
|
||||
)
|
||||
)
|
||||
request_offset += num_requests
|
||||
boundary_offset += num_requests + 1
|
||||
token_offset += num_tokens
|
||||
local_token_offset += num_local_tokens
|
||||
|
||||
seq_lens_cpu = _flat_int32(seq_lens_flat)
|
||||
starts = _flat_int32(starts_flat).to(device, non_blocking=True)
|
||||
cu_seq_lens = _flat_int32(cu_seq_lens_flat).to(device, non_blocking=True)
|
||||
cu_seqlens_q = _flat_int32(cu_seqlens_q_flat).to(device, non_blocking=True)
|
||||
token_to_seq = _flat_int32(np.concatenate(token_to_seq_parts)).to(
|
||||
device, non_blocking=True
|
||||
)
|
||||
if use_dcp:
|
||||
padded_local_cu_seq_lens = _flat_int32(padded_local_cu_seq_lens_flat).to(
|
||||
device, non_blocking=True
|
||||
)
|
||||
padded_local_token_to_seq = _flat_int32(
|
||||
np.concatenate(padded_local_token_to_seq_parts)
|
||||
).to(device, non_blocking=True)
|
||||
|
||||
chunks: list[MLACommonPrefillMetadata.ContextChunk] = []
|
||||
for index, (plan, layout) in enumerate(zip(plans, layouts)):
|
||||
request_slice, boundary_slice, token_slice, local_token_slice = layout
|
||||
query_lens = [
|
||||
query_start_loc[request + 1] - query_start_loc[request]
|
||||
for request in range(plan.request_start, plan.request_end)
|
||||
]
|
||||
chunk = MLACommonPrefillMetadata.ContextChunk(
|
||||
index=index,
|
||||
request_slice=slice(plan.request_start, plan.request_end),
|
||||
token_slice=slice(
|
||||
query_start_loc[plan.request_start], query_start_loc[plan.request_end]
|
||||
),
|
||||
continuation_token_end=query_start_loc[plan.request_start + 1],
|
||||
is_continuation=plan.is_continuation,
|
||||
num_context_tokens=token_slice.stop - token_slice.start,
|
||||
query_start_loc=cu_seqlens_q[boundary_slice],
|
||||
max_query_len=max(query_lens),
|
||||
cu_seq_lens=cu_seq_lens[boundary_slice],
|
||||
starts=starts[request_slice],
|
||||
max_seq_len=max(plan.seq_lens),
|
||||
seq_lens=seq_lens_cpu[request_slice],
|
||||
token_to_seq=token_to_seq[token_slice],
|
||||
num_local_context_tokens=local_token_slice.stop - local_token_slice.start,
|
||||
)
|
||||
if use_dcp:
|
||||
assert local_context_lens_allranks is not None
|
||||
chunk.padded_local_seq_lens = local_seq_lens_per_chunk[index]
|
||||
chunk.local_context_lens_allranks = local_context_lens_allranks[
|
||||
plan.request_start : plan.request_end
|
||||
]
|
||||
chunk.padded_local_cu_seq_lens = padded_local_cu_seq_lens[boundary_slice]
|
||||
chunk.padded_local_token_to_seq = padded_local_token_to_seq[
|
||||
local_token_slice
|
||||
]
|
||||
chunk.local_starts = local_starts_per_chunk[index]
|
||||
chunks.append(chunk)
|
||||
|
||||
return MLACommonPrefillMetadata.ChunkedContextMetadata(
|
||||
context_lens=context_lens_cpu.to(device, non_blocking=True),
|
||||
workspace=chunked_prefill_workspace,
|
||||
chunks=chunks,
|
||||
context_lens_list=context_lens,
|
||||
empty_token_slices=empty_token_slices,
|
||||
)
|
||||
|
||||
|
||||
class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
@@ -1796,13 +1954,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
64 * 1024,
|
||||
)
|
||||
|
||||
# Enforce that we enough for at least 1 page per request
|
||||
chunked_prefill_workspace_size = max(
|
||||
chunked_prefill_workspace_size,
|
||||
scheduler_config.max_num_seqs * cache_config.block_size,
|
||||
)
|
||||
|
||||
return chunked_prefill_workspace_size
|
||||
return max(chunked_prefill_workspace_size, cache_config.block_size)
|
||||
|
||||
@staticmethod
|
||||
def determine_prefill_query_data_type(
|
||||
@@ -2063,7 +2215,6 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
chunked_context_metadata = build_mla_chunked_context_metadata(
|
||||
context_lens_cpu=context_lens_cpu,
|
||||
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
|
||||
num_prefills=num_prefills,
|
||||
chunked_prefill_workspace=self.chunked_prefill_workspace,
|
||||
chunked_prefill_workspace_size=self.chunked_prefill_workspace_size,
|
||||
block_size=self.page_size,
|
||||
@@ -2138,10 +2289,9 @@ def reorg_kvcache(
|
||||
allgatered_k_pe: torch.Tensor,
|
||||
padded_local_chunk_seq_lens_lst: list[int],
|
||||
local_context_lens_allranks: list[list[int]],
|
||||
local_starts: list[int],
|
||||
sum_seq_len: int,
|
||||
max_seq_len: int,
|
||||
chunk_size: int,
|
||||
chunk_idx: int,
|
||||
toks: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
@@ -2155,19 +2305,18 @@ def reorg_kvcache(
|
||||
padded_local_chunk_seq_lens_lst: local chunk context lengths
|
||||
under current CP rank.
|
||||
local_context_lens_allranks: local context lengths on each CP rank.
|
||||
local_starts: per-request local offset into the context this chunk
|
||||
starts at.
|
||||
sum_seq_len: the sum of cp_chunk_seq_lens_lst.
|
||||
max_seq_len: the max value of cp_chunk_seq_lens_lst.
|
||||
chunk_size: the local padded max context chunk from
|
||||
chunked_context_metadata building.
|
||||
chunk_idx: chunk idx of chunked_prefill.
|
||||
toks: the number of tokens for local gather cache.
|
||||
"""
|
||||
kv_c_segments = []
|
||||
k_pe_segments = []
|
||||
src_token_idx = 0
|
||||
max_seq_len_check = 0
|
||||
for padded_local_chunk_seq_len, local_context_lens in zip(
|
||||
padded_local_chunk_seq_lens_lst, local_context_lens_allranks
|
||||
for padded_local_chunk_seq_len, local_context_lens, local_start in zip(
|
||||
padded_local_chunk_seq_lens_lst, local_context_lens_allranks, local_starts
|
||||
):
|
||||
cur_seq_len = 0
|
||||
for rank, local_context_len in enumerate(local_context_lens):
|
||||
@@ -2179,7 +2328,7 @@ def reorg_kvcache(
|
||||
# local_chunk_len in dcp1: |-----|-----|--|
|
||||
# so we need update the last chunk length in dcp1.
|
||||
local_chunk_len = min(
|
||||
max(0, local_context_len - chunk_idx * chunk_size),
|
||||
max(0, local_context_len - local_start),
|
||||
padded_local_chunk_seq_len,
|
||||
)
|
||||
if local_chunk_len != 0:
|
||||
@@ -2206,6 +2355,66 @@ def reorg_kvcache(
|
||||
return reorganized_kv_c_normed, reorganized_k_pe
|
||||
|
||||
|
||||
def init_mla_context_partial(
|
||||
chunked_context: "MLACommonPrefillMetadata.ChunkedContextMetadata",
|
||||
attn_output: torch.Tensor,
|
||||
attn_softmax_lse: torch.Tensor,
|
||||
num_tokens: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Allocate the running context partial over all prefill tokens.
|
||||
|
||||
Laid out like the chunk partials so the final whole-batch merge against the
|
||||
suffix partial sees matching head strides.
|
||||
"""
|
||||
output = torch.empty(
|
||||
(num_tokens, *attn_output.shape[1:]),
|
||||
dtype=attn_output.dtype,
|
||||
device=attn_output.device,
|
||||
)
|
||||
output_lse = torch.empty(
|
||||
(attn_softmax_lse.shape[0], num_tokens),
|
||||
dtype=attn_softmax_lse.dtype,
|
||||
device=attn_softmax_lse.device,
|
||||
)
|
||||
# No chunk covers a prefill without context, so neutralize its partial.
|
||||
for token_slice in chunked_context.empty_token_slices:
|
||||
output[token_slice].zero_()
|
||||
output_lse[:, token_slice].fill_(float("-inf"))
|
||||
return output, output_lse
|
||||
|
||||
|
||||
def accumulate_mla_context_chunk(
|
||||
chunk: "MLACommonPrefillMetadata.ContextChunk",
|
||||
attn_output: torch.Tensor,
|
||||
attn_softmax_lse: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
output_lse: torch.Tensor,
|
||||
) -> None:
|
||||
"""Fold one chunk's partial into the running context partial.
|
||||
|
||||
Only the first request may be a continuation; its tokens are merged and the
|
||||
remaining token range is initialized.
|
||||
"""
|
||||
token_start = chunk.token_slice.start
|
||||
token_end = chunk.token_slice.stop
|
||||
init_start = token_start
|
||||
if chunk.is_continuation:
|
||||
init_start = chunk.continuation_token_end
|
||||
num_merged = init_start - token_start
|
||||
merge_attn_states(
|
||||
output=output[token_start:init_start],
|
||||
output_lse=output_lse[:, token_start:init_start],
|
||||
prefix_output=output[token_start:init_start],
|
||||
prefix_lse=output_lse[:, token_start:init_start],
|
||||
suffix_output=attn_output[:num_merged],
|
||||
suffix_lse=attn_softmax_lse[:, :num_merged],
|
||||
)
|
||||
if init_start < token_end:
|
||||
written = init_start - token_start
|
||||
output[init_start:token_end].copy_(attn_output[written:])
|
||||
output_lse[:, init_start:token_end].copy_(attn_softmax_lse[:, written:])
|
||||
|
||||
|
||||
class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
"""
|
||||
Shared MLA base providing dense-MHA prefill (via the selected
|
||||
@@ -2282,53 +2491,53 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
assert attn_metadata.prefill is not None
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
assert prefill_metadata.prefill_backend is not None
|
||||
assert prefill_metadata.chunked_context is not None
|
||||
chunked_context = prefill_metadata.chunked_context
|
||||
assert chunked_context is not None
|
||||
|
||||
use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype()
|
||||
kv_b_proj_input_dtype = _get_kv_b_proj_input_dtype(
|
||||
self.kv_b_proj, use_fp8_prefill
|
||||
)
|
||||
|
||||
output = None
|
||||
merge_output = None
|
||||
iters = len(prefill_metadata.chunked_context.seq_tot)
|
||||
workspace = prefill_metadata.chunked_context.workspace
|
||||
workspace = chunked_context.workspace
|
||||
|
||||
if use_fp8_prefill:
|
||||
q = q.to(prefill_metadata.q_data_type)
|
||||
|
||||
for i in range(iters):
|
||||
toks = prefill_metadata.chunked_context.seq_tot[i]
|
||||
output = None
|
||||
output_lse = None
|
||||
for chunk in chunked_context.chunks:
|
||||
toks = chunk.num_context_tokens
|
||||
block_table = prefill_metadata.block_table[chunk.request_slice]
|
||||
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=prefill_metadata.block_table,
|
||||
workspace_starts=prefill_metadata.chunked_context.cu_seq_lens[i],
|
||||
batch_size=attn_metadata.num_prefills,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
block_table=block_table,
|
||||
workspace_starts=chunk.cu_seq_lens,
|
||||
batch_size=chunk.num_requests,
|
||||
seq_starts=chunk.starts,
|
||||
)
|
||||
elif not use_fp8_prefill:
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
cu_seq_lens=prefill_metadata.chunked_context.cu_seq_lens[i],
|
||||
token_to_seq=prefill_metadata.chunked_context.token_to_seq[i],
|
||||
num_tokens=prefill_metadata.chunked_context.chunk_total_token[i],
|
||||
block_table=block_table,
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
token_to_seq=chunk.token_to_seq,
|
||||
num_tokens=toks,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
scale=k_scale,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
seq_starts=chunk.starts,
|
||||
)
|
||||
else:
|
||||
# FP8 path: gather cache without dequantization
|
||||
ops.cp_gather_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
cu_seq_lens=prefill_metadata.chunked_context.cu_seq_lens[i],
|
||||
batch_size=attn_metadata.num_prefills,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
block_table=block_table,
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
batch_size=chunk.num_requests,
|
||||
seq_starts=chunk.starts,
|
||||
)
|
||||
|
||||
# Extract kv_c_normed from workspace
|
||||
@@ -2351,37 +2560,28 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
|
||||
attn_output, attn_softmax_lse = (
|
||||
prefill_metadata.prefill_backend.run_prefill_context_chunk(
|
||||
chunk_idx=i,
|
||||
q=q,
|
||||
chunk=chunk,
|
||||
q=q[chunk.token_slice],
|
||||
k=k,
|
||||
v=v,
|
||||
)
|
||||
)
|
||||
if prefill_metadata.chunked_context.has_empty_context[i]:
|
||||
mask_empty_context(
|
||||
attn_softmax_lse,
|
||||
attn_output,
|
||||
prefill_metadata.query_start_loc,
|
||||
prefill_metadata.chunked_context.cu_seq_lens[i],
|
||||
)
|
||||
|
||||
if output is None:
|
||||
output = attn_output
|
||||
output_lse = attn_softmax_lse
|
||||
else:
|
||||
if merge_output is None:
|
||||
merge_output = torch.empty_like(output)
|
||||
merge_output_lse = torch.empty_like(output_lse)
|
||||
merge_attn_states(
|
||||
output=merge_output,
|
||||
output_lse=merge_output_lse,
|
||||
prefix_output=output,
|
||||
prefix_lse=output_lse,
|
||||
suffix_output=attn_output,
|
||||
suffix_lse=attn_softmax_lse,
|
||||
if (
|
||||
len(chunked_context.chunks) == 1
|
||||
and not chunked_context.empty_token_slices
|
||||
):
|
||||
return attn_output, attn_softmax_lse
|
||||
output, output_lse = init_mla_context_partial(
|
||||
chunked_context,
|
||||
attn_output,
|
||||
attn_softmax_lse,
|
||||
num_tokens=q.shape[0],
|
||||
)
|
||||
output, merge_output = merge_output, output
|
||||
output_lse, merge_output_lse = merge_output_lse, output_lse
|
||||
accumulate_mla_context_chunk(
|
||||
chunk, attn_output, attn_softmax_lse, output, output_lse
|
||||
)
|
||||
|
||||
return output, output_lse
|
||||
|
||||
@@ -2396,62 +2596,57 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
assert attn_metadata.prefill is not None
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
assert prefill_metadata.prefill_backend is not None
|
||||
assert prefill_metadata.chunked_context is not None
|
||||
assert prefill_metadata.chunked_context.padded_local_chunk_seq_lens is not None
|
||||
assert prefill_metadata.chunked_context.local_context_lens_allranks is not None
|
||||
assert prefill_metadata.chunked_context.padded_local_cu_seq_lens is not None
|
||||
assert prefill_metadata.chunked_context.padded_local_token_to_seq is not None
|
||||
assert prefill_metadata.chunked_context.cu_seq_lens_lst is not None
|
||||
assert prefill_metadata.chunked_context.chunk_size is not None
|
||||
chunked_context = prefill_metadata.chunked_context
|
||||
assert chunked_context is not None
|
||||
|
||||
use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype()
|
||||
kv_b_proj_input_dtype = _get_kv_b_proj_input_dtype(
|
||||
self.kv_b_proj, use_fp8_prefill
|
||||
)
|
||||
output = None
|
||||
merge_output = None
|
||||
iters = len(prefill_metadata.chunked_context.seq_tot)
|
||||
workspace = prefill_metadata.chunked_context.workspace
|
||||
output_lse = None
|
||||
workspace = chunked_context.workspace
|
||||
|
||||
for i in range(iters):
|
||||
toks = prefill_metadata.chunked_context.seq_tot[i]
|
||||
if toks == 0:
|
||||
continue
|
||||
padded_local_cu_seq_lens = (
|
||||
prefill_metadata.chunked_context.padded_local_cu_seq_lens[i]
|
||||
)
|
||||
for chunk in chunked_context.chunks:
|
||||
assert chunk.padded_local_seq_lens is not None
|
||||
assert chunk.local_context_lens_allranks is not None
|
||||
assert chunk.padded_local_cu_seq_lens is not None
|
||||
assert chunk.padded_local_token_to_seq is not None
|
||||
assert chunk.local_starts is not None
|
||||
|
||||
toks = chunk.num_local_context_tokens
|
||||
padded_local_cu_seq_lens = chunk.padded_local_cu_seq_lens
|
||||
block_table = prefill_metadata.block_table[chunk.request_slice]
|
||||
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=prefill_metadata.block_table,
|
||||
block_table=block_table,
|
||||
workspace_starts=padded_local_cu_seq_lens,
|
||||
batch_size=attn_metadata.num_prefills,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
batch_size=chunk.num_requests,
|
||||
seq_starts=chunk.starts,
|
||||
)
|
||||
elif is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
assert k_scale is not None
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
block_table=block_table,
|
||||
cu_seq_lens=padded_local_cu_seq_lens,
|
||||
token_to_seq=prefill_metadata.chunked_context.padded_local_token_to_seq[
|
||||
i
|
||||
],
|
||||
token_to_seq=chunk.padded_local_token_to_seq,
|
||||
num_tokens=toks,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
scale=k_scale,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
seq_starts=chunk.starts,
|
||||
)
|
||||
else:
|
||||
ops.cp_gather_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
block_table=block_table,
|
||||
cu_seq_lens=padded_local_cu_seq_lens,
|
||||
batch_size=attn_metadata.num_prefills,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
batch_size=chunk.num_requests,
|
||||
seq_starts=chunk.starts,
|
||||
)
|
||||
# workspace
|
||||
# |------- N tokens --------|--------- N*dcp_size tokens ----------|
|
||||
@@ -2479,14 +2674,11 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
kv_c_normed, k_pe = reorg_kvcache(
|
||||
allgatered_kv_c_normed,
|
||||
allgatered_k_pe,
|
||||
padded_local_chunk_seq_lens_lst=prefill_metadata.chunked_context.padded_local_chunk_seq_lens[
|
||||
i
|
||||
],
|
||||
local_context_lens_allranks=prefill_metadata.chunked_context.local_context_lens_allranks,
|
||||
sum_seq_len=prefill_metadata.chunked_context.cu_seq_lens_lst[i][-1],
|
||||
max_seq_len=prefill_metadata.chunked_context.max_seq_lens[i],
|
||||
chunk_size=prefill_metadata.chunked_context.chunk_size,
|
||||
chunk_idx=i,
|
||||
padded_local_chunk_seq_lens_lst=chunk.padded_local_seq_lens,
|
||||
local_context_lens_allranks=chunk.local_context_lens_allranks,
|
||||
local_starts=chunk.local_starts,
|
||||
sum_seq_len=chunk.num_context_tokens,
|
||||
max_seq_len=chunk.max_seq_len,
|
||||
toks=toks,
|
||||
)
|
||||
if kv_b_proj_input_dtype is not None:
|
||||
@@ -2503,37 +2695,28 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
|
||||
attn_output, attn_softmax_lse = (
|
||||
prefill_metadata.prefill_backend.run_prefill_context_chunk(
|
||||
chunk_idx=i,
|
||||
q=q,
|
||||
chunk=chunk,
|
||||
q=q[chunk.token_slice],
|
||||
k=k,
|
||||
v=v,
|
||||
)
|
||||
)
|
||||
if prefill_metadata.chunked_context.has_empty_context[i]:
|
||||
mask_empty_context(
|
||||
attn_softmax_lse,
|
||||
attn_output,
|
||||
prefill_metadata.query_start_loc,
|
||||
prefill_metadata.chunked_context.cu_seq_lens[i],
|
||||
)
|
||||
|
||||
if output is None:
|
||||
output = attn_output
|
||||
output_lse = attn_softmax_lse
|
||||
else:
|
||||
if merge_output is None:
|
||||
merge_output = torch.empty_like(output)
|
||||
merge_output_lse = torch.empty_like(output_lse)
|
||||
merge_attn_states(
|
||||
output=merge_output,
|
||||
output_lse=merge_output_lse,
|
||||
prefix_output=output,
|
||||
prefix_lse=output_lse,
|
||||
suffix_output=attn_output,
|
||||
suffix_lse=attn_softmax_lse,
|
||||
if (
|
||||
len(chunked_context.chunks) == 1
|
||||
and not chunked_context.empty_token_slices
|
||||
):
|
||||
return attn_output, attn_softmax_lse
|
||||
output, output_lse = init_mla_context_partial(
|
||||
chunked_context,
|
||||
attn_output,
|
||||
attn_softmax_lse,
|
||||
num_tokens=q.shape[0],
|
||||
)
|
||||
output, merge_output = merge_output, output
|
||||
output_lse, merge_output_lse = merge_output_lse, output_lse
|
||||
accumulate_mla_context_chunk(
|
||||
chunk, attn_output, attn_softmax_lse, output, output_lse
|
||||
)
|
||||
|
||||
return output, output_lse
|
||||
|
||||
@@ -2615,7 +2798,6 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
prefix_lse=context_lse,
|
||||
suffix_output=suffix_output,
|
||||
suffix_lse=suffix_lse,
|
||||
prefill_tokens_with_context=prefill_metadata.chunked_context.prefill_tokens_with_context,
|
||||
)
|
||||
elif output_scale is None:
|
||||
# With output_scale set, backend already wrote into `output` in place.
|
||||
|
||||
@@ -19,8 +19,10 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBaseImpl,
|
||||
MLACommonMetadata,
|
||||
MLACommonPrefillMetadata,
|
||||
accumulate_mla_context_chunk,
|
||||
build_mla_chunked_context_metadata,
|
||||
get_mla_dims,
|
||||
init_mla_context_partial,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
@@ -60,17 +62,14 @@ def _topk_mask_shape(
|
||||
def _masked_mha_workspace_fits(
|
||||
batch_size: int,
|
||||
max_query_len: int,
|
||||
context_chunk_max_seq_lens: list[int] | None,
|
||||
max_context_chunk_seq_len: int,
|
||||
workspace_numel: int,
|
||||
) -> bool:
|
||||
"""Return whether the suffix and per-context-chunk masks fit the workspace.
|
||||
|
||||
The global mask is excluded: it always needs more, and has its own check.
|
||||
"""
|
||||
max_key_len = max(
|
||||
max_query_len,
|
||||
max(context_chunk_max_seq_lens or (), default=0),
|
||||
)
|
||||
max_key_len = max(max_query_len, max_context_chunk_seq_len)
|
||||
needed = math.prod(_topk_mask_shape(batch_size, max_query_len, max_key_len))
|
||||
return needed <= workspace_numel
|
||||
|
||||
@@ -185,10 +184,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
64 * 1024,
|
||||
scheduler_config.max_num_seqs * topk_tokens,
|
||||
)
|
||||
return max(
|
||||
workspace_size,
|
||||
scheduler_config.max_num_seqs * cache_config.block_size,
|
||||
)
|
||||
return max(workspace_size, cache_config.block_size)
|
||||
|
||||
def _build_req_id_per_token(
|
||||
self,
|
||||
@@ -228,7 +224,6 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
return build_mla_chunked_context_metadata(
|
||||
context_lens_cpu=context_lens_cpu,
|
||||
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
|
||||
num_prefills=num_prefills,
|
||||
chunked_prefill_workspace=self.chunked_prefill_workspace,
|
||||
chunked_prefill_workspace_size=self.chunked_prefill_workspace_size,
|
||||
block_size=self.kv_cache_spec.block_size,
|
||||
@@ -542,14 +537,15 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
workspace = prefill.topk_mask_workspace
|
||||
if workspace is None or prefill.query_lens_cpu is None:
|
||||
return False
|
||||
max_context_chunk_seq_len = 0
|
||||
if prefill.chunked_context is not None:
|
||||
max_context_chunk_seq_len = max(
|
||||
chunk.max_seq_len for chunk in prefill.chunked_context.chunks
|
||||
)
|
||||
fits = _masked_mha_workspace_fits(
|
||||
batch_size=len(prefill.query_lens_cpu),
|
||||
max_query_len=prefill.max_query_len,
|
||||
context_chunk_max_seq_lens=(
|
||||
None
|
||||
if prefill.chunked_context is None
|
||||
else prefill.chunked_context.max_seq_lens
|
||||
),
|
||||
max_context_chunk_seq_len=max_context_chunk_seq_len,
|
||||
workspace_numel=workspace.numel(),
|
||||
)
|
||||
if not fits:
|
||||
@@ -716,69 +712,70 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
|
||||
chunked_context = prefill_metadata.chunked_context
|
||||
assert chunked_context is not None
|
||||
use_global_mask = dense_mask is not None
|
||||
output: torch.Tensor | None = None
|
||||
output_lse: torch.Tensor | None = None
|
||||
workspace = chunked_context.workspace
|
||||
|
||||
for i, toks in enumerate(chunked_context.seq_tot):
|
||||
if toks == 0:
|
||||
continue
|
||||
for chunk in chunked_context.chunks:
|
||||
toks = chunk.num_context_tokens
|
||||
requests = chunk.request_slice
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
cu_seq_lens=chunked_context.cu_seq_lens[i],
|
||||
token_to_seq=chunked_context.token_to_seq[i],
|
||||
num_tokens=chunked_context.chunk_total_token[i],
|
||||
block_table=prefill_metadata.block_table[requests],
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
token_to_seq=chunk.token_to_seq,
|
||||
num_tokens=toks,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
scale=k_scale,
|
||||
seq_starts=chunked_context.starts[i],
|
||||
seq_starts=chunk.starts,
|
||||
)
|
||||
|
||||
chunk_kv_c = workspace[:toks, : self.kv_lora_rank]
|
||||
chunk_k_pe = workspace[:toks, self.kv_lora_rank :].unsqueeze(1)
|
||||
k, v = self._project_kv(chunk_kv_c, chunk_k_pe)
|
||||
chunk_lens = chunked_context.seq_lens[i].tolist()
|
||||
chunk_topk = (
|
||||
topk_per_req
|
||||
if use_global_mask
|
||||
else self._remap_topk_to_ranges(
|
||||
topk_per_req,
|
||||
chunked_context.starts[i],
|
||||
chunk_lens,
|
||||
if dense_mask is not None:
|
||||
chunk_mask: torch.Tensor | None = dense_mask[requests]
|
||||
chunk_topk = topk_per_req[requests]
|
||||
key_starts: torch.Tensor | None = chunk.starts
|
||||
else:
|
||||
chunk_mask = None
|
||||
chunk_topk = self._remap_topk_to_ranges(
|
||||
topk_per_req[requests],
|
||||
chunk.starts,
|
||||
chunk.seq_lens.tolist(),
|
||||
)
|
||||
)
|
||||
key_starts = None
|
||||
attn_out, lse = self._run_masked_mha(
|
||||
q=q,
|
||||
q=q[chunk.token_slice],
|
||||
k=k,
|
||||
v=v,
|
||||
cu_seqlens_q=prefill_metadata.query_start_loc,
|
||||
cu_seqlens_k=chunked_context.cu_seq_lens[i],
|
||||
max_seqlen_q=prefill_metadata.max_query_len,
|
||||
max_seqlen_k=chunked_context.max_seq_lens[i],
|
||||
cu_seqlens_q=chunk.query_start_loc,
|
||||
cu_seqlens_k=chunk.cu_seq_lens,
|
||||
max_seqlen_q=chunk.max_query_len,
|
||||
max_seqlen_k=chunk.max_seq_len,
|
||||
topk_per_req=chunk_topk,
|
||||
q_lens=q_lens,
|
||||
q_lens=q_lens[requests],
|
||||
causal=False,
|
||||
return_softmax_lse=True,
|
||||
dense_mask=dense_mask,
|
||||
key_starts=(chunked_context.starts[i] if use_global_mask else None),
|
||||
dense_mask=chunk_mask,
|
||||
key_starts=key_starts,
|
||||
topk_mask_workspace=prefill_metadata.topk_mask_workspace,
|
||||
)
|
||||
|
||||
if output is None:
|
||||
output = attn_out
|
||||
output_lse = lse
|
||||
else:
|
||||
assert output_lse is not None
|
||||
merge_attn_states(
|
||||
output=output,
|
||||
output_lse=output_lse,
|
||||
prefix_output=output,
|
||||
prefix_lse=output_lse,
|
||||
suffix_output=attn_out,
|
||||
suffix_lse=lse,
|
||||
if (
|
||||
len(chunked_context.chunks) == 1
|
||||
and not chunked_context.empty_token_slices
|
||||
):
|
||||
return attn_out, lse
|
||||
output, output_lse = init_mla_context_partial(
|
||||
chunked_context,
|
||||
attn_out,
|
||||
lse,
|
||||
num_tokens=q.shape[0],
|
||||
)
|
||||
accumulate_mla_context_chunk(chunk, attn_out, lse, output, output_lse)
|
||||
|
||||
assert output is not None and output_lse is not None
|
||||
return output, output_lse
|
||||
@@ -844,7 +841,7 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
output.copy_(attn_out[..., : self.v_head_dim].flatten(start_dim=-2))
|
||||
return
|
||||
|
||||
context_lens = chunked_context.seq_lens.sum(dim=0).tolist()
|
||||
context_lens = chunked_context.context_lens_list
|
||||
dense_mask = self._try_build_global_mask(
|
||||
topk_per_req,
|
||||
q_lens,
|
||||
@@ -889,5 +886,4 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
prefix_lse=context_lse,
|
||||
suffix_output=suffix_output[..., : self.v_head_dim],
|
||||
suffix_lse=suffix_lse,
|
||||
prefill_tokens_with_context=chunked_context.prefill_tokens_with_context,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user