[1/N] Add dense MHA path for sparse MLA short sequences (#47327)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
0b0ef8d7eb
commit
c4f5cd60da
@@ -259,7 +259,6 @@ from vllm.v1.attention.backend import (
|
||||
AttentionType,
|
||||
CommonAttentionMetadata,
|
||||
MLAAttentionImpl,
|
||||
SparseMLAAttentionImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill import (
|
||||
MLAPrefillBackend,
|
||||
@@ -490,16 +489,30 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
self.prefill_backend = prefill_backend_cls(
|
||||
num_heads=self.num_heads,
|
||||
scale=self.scale,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
self.prefill_backend: MLAPrefillBackend | None
|
||||
try:
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
except ValueError:
|
||||
if (
|
||||
not self.impl.is_sparse
|
||||
or vllm_config.attention_config.mla_prefill_backend is not None
|
||||
):
|
||||
raise
|
||||
logger.warning_once(
|
||||
"No MLA prefill backend supports this model; sparse MLA will use the "
|
||||
"top-k MQA path only (no dense-MHA prefill)."
|
||||
)
|
||||
self.prefill_backend = None
|
||||
else:
|
||||
self.prefill_backend = prefill_backend_cls(
|
||||
num_heads=self.num_heads,
|
||||
scale=self.scale,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
self.kv_cache = torch.tensor([])
|
||||
|
||||
@@ -693,23 +706,28 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
if fp8_attention and self.kv_cache_dtype != "fp8_ds_mla":
|
||||
kv_cache = kv_cache.view(current_platform.fp8_dtype())
|
||||
|
||||
# Sparse MLA impls only support forward_mqa (decode-style attention)
|
||||
is_sparse_impl = isinstance(self.impl, SparseMLAAttentionImpl)
|
||||
assert (
|
||||
attn_metadata.num_decodes is not None
|
||||
and attn_metadata.num_prefills is not None
|
||||
and attn_metadata.num_decode_tokens is not None
|
||||
)
|
||||
num_mqa_tokens = attn_metadata.num_decode_tokens
|
||||
num_mha_tokens = q.size(0) - num_mqa_tokens
|
||||
|
||||
if is_sparse_impl:
|
||||
num_mqa_tokens = q.size(0)
|
||||
num_mha_tokens = 0
|
||||
else:
|
||||
assert (
|
||||
attn_metadata.num_decodes is not None
|
||||
and attn_metadata.num_prefills is not None
|
||||
and attn_metadata.num_decode_tokens is not None
|
||||
if self.impl.is_sparse and num_mha_tokens > 0:
|
||||
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
|
||||
use_mha = (
|
||||
self.prefill_backend is not None
|
||||
and prefill_max_seq_len <= attn_metadata.topk_tokens # type: ignore[attr-defined]
|
||||
and not self._vllm_config.attention_config.sparse_mla_force_mqa
|
||||
)
|
||||
num_mqa_tokens = attn_metadata.num_decode_tokens
|
||||
num_mha_tokens = q.size(0) - num_mqa_tokens
|
||||
if not use_mha:
|
||||
num_mqa_tokens = q.size(0)
|
||||
num_mha_tokens = 0
|
||||
|
||||
mha_use_quant_output = (
|
||||
quant_key is not None
|
||||
and self.prefill_backend is not None
|
||||
and self.prefill_backend.supports_quant_output(quant_key)
|
||||
and attn_metadata is not None
|
||||
and attn_metadata.prefill is not None
|
||||
@@ -807,7 +825,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
|
||||
|
||||
# call decode attn
|
||||
if not is_sparse_impl:
|
||||
if not self.impl.is_sparse:
|
||||
assert attn_metadata.decode is not None
|
||||
attn_out, lse = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self) # type: ignore[attr-defined]
|
||||
|
||||
@@ -1398,6 +1416,182 @@ def backend_supports_prefill_query_quantization() -> bool:
|
||||
)
|
||||
|
||||
|
||||
def build_mla_chunked_context_metadata(
|
||||
*,
|
||||
context_lens_cpu: torch.Tensor,
|
||||
prefill_query_start_loc_cpu: torch.Tensor,
|
||||
num_prefills: int,
|
||||
chunked_prefill_workspace: torch.Tensor,
|
||||
chunked_prefill_workspace_size: int,
|
||||
block_size: int,
|
||||
align_chunk_to_block: bool,
|
||||
device: torch.device,
|
||||
dcp_world_size: int,
|
||||
dcp_local_block_size: int,
|
||||
dcp_virtual_block_size: int,
|
||||
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
|
||||
"""Build chunked-context metadata for an MLA prefill.
|
||||
|
||||
Shared by dense and sparse builders. Splits each prefill's context
|
||||
into workspace-sized chunks and, under DCP, plans the per-rank interleaved
|
||||
local chunks the all-gather reduction consumes.
|
||||
|
||||
Args:
|
||||
context_lens_cpu: Per-prefill context length (seq_len - query_len).
|
||||
prefill_query_start_loc_cpu: Prefill query cumulative offsets (0-based).
|
||||
num_prefills: Number of prefill requests.
|
||||
chunked_prefill_workspace: Scratch buffer the context gather writes to.
|
||||
chunked_prefill_workspace_size: Row capacity of the workspace.
|
||||
block_size: KV cache page size for chunk-start alignment.
|
||||
align_chunk_to_block: Round the chunk size down to ``block_size``.
|
||||
device: Target device for the returned tensors.
|
||||
dcp_world_size: Decode-context-parallel world size (1 if disabled).
|
||||
dcp_local_block_size: Per-rank interleave block size for DCP.
|
||||
dcp_virtual_block_size: ``dcp_local_block_size * dcp_world_size``.
|
||||
|
||||
Returns:
|
||||
The chunked-context metadata, or None when no prefill has any context.
|
||||
"""
|
||||
# NOTE: it is recommended you read the `Chunked Prefill` section in the
|
||||
# comment at the top of the file before trying to understand this code.
|
||||
max_context_len = context_lens_cpu.max().item()
|
||||
if max_context_len <= 0:
|
||||
return None
|
||||
num_prefills_with_context = int((context_lens_cpu > 0).sum().item())
|
||||
|
||||
# Currently we allocate an equal amount of workspace for each prefill with
|
||||
# context; we could probably use a more advanced algorithm here and allocate
|
||||
# more workspace to prefills with longer context lengths.
|
||||
max_context_chunk = chunked_prefill_workspace_size // num_prefills_with_context
|
||||
if align_chunk_to_block:
|
||||
# The `gather_and_maybe_dequant_cache` kernel cannot handle chunk
|
||||
# starts that are not aligned to block_size, so round down.
|
||||
max_context_chunk = round_down(max_context_chunk, block_size)
|
||||
assert max_context_chunk > 0
|
||||
|
||||
num_chunks = cdiv(max_context_len, max_context_chunk)
|
||||
# e.g. max_context_chunk=256, num_chunks=3, num_prefills=4 ->
|
||||
# [[0, 0, 0, 0], [256, 256, 256, 256], [512, 512, 512, 512]]
|
||||
# Note(simon): this is done on CPU because of downstream's use of `to_list`.
|
||||
chunk_starts = torch.empty(
|
||||
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
||||
).copy_(
|
||||
torch.arange(num_chunks, dtype=torch.int32)
|
||||
.multiply_(max_context_chunk)
|
||||
.unsqueeze(1)
|
||||
)
|
||||
chunk_ends = torch.min(
|
||||
context_lens_cpu.unsqueeze(0), chunk_starts + max_context_chunk
|
||||
)
|
||||
chunk_seq_lens = (chunk_ends - chunk_starts).clamp(min=0)
|
||||
|
||||
cu_seq_lens_cpu = torch.zeros(
|
||||
num_chunks, num_prefills + 1, dtype=torch.int32, pin_memory=True
|
||||
)
|
||||
torch.cumsum(chunk_seq_lens, dim=1, out=cu_seq_lens_cpu[:, 1:], dtype=torch.int32)
|
||||
chunk_total_token = cu_seq_lens_cpu[:, -1]
|
||||
|
||||
max_tokens_over_chunk = chunk_total_token.max().item()
|
||||
token_to_seq_cpu = torch.zeros(
|
||||
(num_chunks, max_tokens_over_chunk), dtype=torch.int32, pin_memory=True
|
||||
)
|
||||
req_indices = torch.arange(num_prefills, dtype=torch.int32)
|
||||
for i in range(num_chunks):
|
||||
token_to_seq = torch.repeat_interleave(req_indices, chunk_seq_lens[i])
|
||||
token_to_seq_cpu[i, : token_to_seq.shape[0]] = token_to_seq
|
||||
|
||||
prefill_tokens_with_context = prefill_query_start_loc_cpu[
|
||||
num_prefills_with_context
|
||||
].item()
|
||||
|
||||
metadata_cls = MLACommonPrefillMetadata.ChunkedContextMetadata
|
||||
if dcp_world_size > 1:
|
||||
local_context_lens_allranks = get_dcp_local_seq_lens(
|
||||
context_lens_cpu, dcp_world_size, None, dcp_local_block_size
|
||||
)
|
||||
# Note(qcs): Per-rank local context lengths, padded to
|
||||
# `dcp_local_block_size`.
|
||||
padded_local_context_lens_cpu: torch.Tensor = (
|
||||
cdiv(context_lens_cpu, dcp_virtual_block_size) * dcp_local_block_size
|
||||
)
|
||||
# Note(hc): The above max_context_chunk already enforces block_size
|
||||
# alignment; DCP only requires block_size be divisible by dcp_world_size,
|
||||
# because DCP uses cp_gather_cache, which does not require chunk starts
|
||||
# aligned to block_size.
|
||||
assert max_context_chunk % dcp_world_size == 0
|
||||
padded_local_max_context_chunk = (
|
||||
cdiv(max_context_chunk, dcp_virtual_block_size) * dcp_local_block_size
|
||||
)
|
||||
local_chunk_starts = torch.empty(
|
||||
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
||||
).copy_(
|
||||
torch.arange(num_chunks, dtype=torch.int32)
|
||||
.multiply_(padded_local_max_context_chunk)
|
||||
.unsqueeze(1)
|
||||
)
|
||||
local_chunk_ends = torch.min(
|
||||
padded_local_context_lens_cpu.unsqueeze(0),
|
||||
local_chunk_starts + padded_local_max_context_chunk,
|
||||
)
|
||||
padded_local_chunk_seq_lens = (local_chunk_ends - local_chunk_starts).clamp(
|
||||
min=0
|
||||
)
|
||||
|
||||
padded_local_cu_seq_lens_cpu = torch.zeros(
|
||||
num_chunks, num_prefills + 1, dtype=torch.int32, pin_memory=True
|
||||
)
|
||||
torch.cumsum(
|
||||
padded_local_chunk_seq_lens,
|
||||
dim=1,
|
||||
out=padded_local_cu_seq_lens_cpu[:, 1:],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
max_padded_local_tokens = padded_local_cu_seq_lens_cpu[:, -1].max().item()
|
||||
padded_local_token_to_seq_cpu = torch.zeros(
|
||||
(num_chunks, max_padded_local_tokens), dtype=torch.int32
|
||||
)
|
||||
for i in range(num_chunks):
|
||||
tts = torch.repeat_interleave(req_indices, padded_local_chunk_seq_lens[i])
|
||||
padded_local_token_to_seq_cpu[i, : tts.shape[0]] = tts
|
||||
|
||||
chunked_context_metadata = metadata_cls(
|
||||
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
||||
starts=local_chunk_starts.to(device, non_blocking=True),
|
||||
seq_tot=padded_local_chunk_seq_lens.sum(dim=1).tolist(),
|
||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||
seq_lens=chunk_seq_lens,
|
||||
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
|
||||
chunk_total_token=chunk_total_token.tolist(),
|
||||
workspace=chunked_prefill_workspace,
|
||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||
padded_local_chunk_seq_lens=padded_local_chunk_seq_lens.tolist(),
|
||||
local_context_lens_allranks=local_context_lens_allranks.tolist(),
|
||||
padded_local_cu_seq_lens=padded_local_cu_seq_lens_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
padded_local_token_to_seq=padded_local_token_to_seq_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
cu_seq_lens_lst=cu_seq_lens_cpu.tolist(),
|
||||
chunk_size=padded_local_max_context_chunk,
|
||||
)
|
||||
else:
|
||||
chunked_context_metadata = metadata_cls(
|
||||
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
||||
starts=chunk_starts.to(device, non_blocking=True),
|
||||
seq_tot=chunk_seq_lens.sum(dim=1).tolist(),
|
||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||
seq_lens=chunk_seq_lens,
|
||||
token_to_seq=token_to_seq_cpu.to(device, non_blocking=True),
|
||||
chunk_total_token=chunk_total_token,
|
||||
workspace=chunked_prefill_workspace,
|
||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||
)
|
||||
|
||||
assert max(chunked_context_metadata.max_seq_lens) <= chunked_prefill_workspace_size
|
||||
return chunked_context_metadata
|
||||
|
||||
|
||||
class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
"""
|
||||
NOTE: Please read the comment at the top of the file before trying to
|
||||
@@ -1662,8 +1856,6 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
context_lens_cpu = (
|
||||
seq_lens_cpu[reqs_start:num_reqs] - prefill_query_lens_cpu
|
||||
)
|
||||
max_context_len_cpu = context_lens_cpu.max().item()
|
||||
num_prefills_with_context_cpu = (context_lens_cpu > 0).sum().item()
|
||||
prefill_query_start_loc = (
|
||||
query_start_loc[reqs_start:] - query_start_loc[reqs_start]
|
||||
)
|
||||
@@ -1671,188 +1863,19 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
query_start_loc_cpu[reqs_start:] - query_start_loc_cpu[reqs_start]
|
||||
)
|
||||
|
||||
chunked_context_metadata = None
|
||||
if max_context_len_cpu > 0:
|
||||
# NOTE: it is recommend you read the `Chunked Prefill` section
|
||||
# in the comment at the top of the file before trying to
|
||||
# understand the following code
|
||||
|
||||
# currently we allocate an equal amount of workspace for each
|
||||
# prefill in the batch, we could probably use a more advanced
|
||||
# algorithm here and allocate more workspace to prefills with
|
||||
# longer context lengths
|
||||
max_context_chunk = (
|
||||
self.chunked_prefill_workspace_size // num_prefills_with_context_cpu
|
||||
)
|
||||
|
||||
# align max_context_chunk to page_size by rounding down,
|
||||
# currently the `gather_and_maybe_dequant_cache` kernel
|
||||
# cannot handle `context_chunk_starts` that are not aligned
|
||||
# to page_size
|
||||
max_context_chunk = round_down(max_context_chunk, self.page_size)
|
||||
|
||||
assert max_context_chunk > 0
|
||||
num_chunks = cdiv(max_context_len_cpu, max_context_chunk)
|
||||
|
||||
# if `max_context_chunk = 256`, `num_chunks = 3`, and
|
||||
# `num_prefills_with_context = 4`, create a tensor that looks
|
||||
# like
|
||||
# [[0, 0, 0, 0], [256, 256, 256, 256], [512, 512, 512, 512]]
|
||||
# Note(simon): this is done in CPU because of downstream's
|
||||
# of `to_list`.
|
||||
chunk_starts = torch.empty(
|
||||
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
||||
).copy_(
|
||||
torch.arange(num_chunks, dtype=torch.int32)
|
||||
.multiply_(max_context_chunk)
|
||||
.unsqueeze(1)
|
||||
)
|
||||
chunk_ends = torch.min(
|
||||
context_lens_cpu.unsqueeze(0), chunk_starts + max_context_chunk
|
||||
)
|
||||
chunk_seq_lens = (chunk_ends - chunk_starts).clamp(min=0)
|
||||
|
||||
cu_seq_lens_cpu = torch.zeros(
|
||||
num_chunks, num_prefills + 1, dtype=torch.int32, pin_memory=True
|
||||
)
|
||||
torch.cumsum(
|
||||
chunk_seq_lens, dim=1, out=cu_seq_lens_cpu[:, 1:], dtype=torch.int32
|
||||
)
|
||||
chunk_total_token = cu_seq_lens_cpu[:, -1]
|
||||
|
||||
max_token_num_over_chunk = chunk_total_token.max().item()
|
||||
token_to_seq_tensor_cpu = torch.zeros(
|
||||
[num_chunks, max_token_num_over_chunk],
|
||||
dtype=torch.int32,
|
||||
pin_memory=True,
|
||||
)
|
||||
range_idx = torch.arange(num_prefills, dtype=torch.int32)
|
||||
for i in range(num_chunks):
|
||||
chunk_token_to_seq_tensor = torch.repeat_interleave(
|
||||
range_idx, chunk_seq_lens[i]
|
||||
)
|
||||
chunk_len = chunk_token_to_seq_tensor.shape[0]
|
||||
token_to_seq_tensor_cpu[i, :chunk_len] = chunk_token_to_seq_tensor
|
||||
|
||||
if self.dcp_world_size > 1:
|
||||
local_context_lens_allranks = get_dcp_local_seq_lens(
|
||||
context_lens_cpu,
|
||||
self.dcp_world_size,
|
||||
None,
|
||||
self.dcp_local_block_size,
|
||||
)
|
||||
# Note(qcs): The max local context lengths
|
||||
# padded to `dcp_local_block_size`.
|
||||
padded_local_context_lens_cpu: torch.Tensor = (
|
||||
cdiv(
|
||||
context_lens_cpu,
|
||||
self.dcp_virtual_block_size,
|
||||
)
|
||||
* self.dcp_local_block_size
|
||||
)
|
||||
# Note(hc): The above max_context_chunk already enforces
|
||||
# block_size alignment, DCP just need the block_size can
|
||||
# be divisible by dcp_world_size, because DCP use
|
||||
# cp_gather_cache which not require `cp_chunk_starts`
|
||||
# aligned to page_size.
|
||||
assert max_context_chunk % self.dcp_world_size == 0
|
||||
padded_local_max_context_chunk_across_ranks = (
|
||||
cdiv(
|
||||
max_context_chunk,
|
||||
self.dcp_virtual_block_size,
|
||||
)
|
||||
* self.dcp_local_block_size
|
||||
)
|
||||
local_chunk_starts = torch.empty(
|
||||
num_chunks, num_prefills, dtype=torch.int32, pin_memory=True
|
||||
).copy_(
|
||||
torch.arange(num_chunks, dtype=torch.int32)
|
||||
.multiply_(padded_local_max_context_chunk_across_ranks)
|
||||
.unsqueeze(1)
|
||||
)
|
||||
local_chunk_ends = torch.min(
|
||||
padded_local_context_lens_cpu.unsqueeze(0),
|
||||
local_chunk_starts
|
||||
+ padded_local_max_context_chunk_across_ranks,
|
||||
)
|
||||
padded_local_chunk_seq_lens = (
|
||||
local_chunk_ends - local_chunk_starts
|
||||
).clamp(min=0)
|
||||
|
||||
padded_local_cu_chunk_seq_lens_cpu = torch.zeros(
|
||||
num_chunks, num_prefills + 1, dtype=torch.int32, pin_memory=True
|
||||
)
|
||||
torch.cumsum(
|
||||
padded_local_chunk_seq_lens,
|
||||
dim=1,
|
||||
out=padded_local_cu_chunk_seq_lens_cpu[:, 1:],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
max_padded_local_tokens_over_chunk = (
|
||||
padded_local_cu_chunk_seq_lens_cpu[:, -1].max().item()
|
||||
)
|
||||
padded_local_token_to_seq_tensor_cpu = torch.zeros(
|
||||
[num_chunks, max_padded_local_tokens_over_chunk],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
for i in range(num_chunks):
|
||||
chunk_token_to_seq_tensor = torch.repeat_interleave(
|
||||
range_idx, padded_local_chunk_seq_lens[i]
|
||||
)
|
||||
chunk_len = chunk_token_to_seq_tensor.shape[0]
|
||||
padded_local_token_to_seq_tensor_cpu[i, :chunk_len] = (
|
||||
chunk_token_to_seq_tensor
|
||||
)
|
||||
|
||||
prefill_tokens_with_context = None
|
||||
if num_prefills_with_context_cpu > 0:
|
||||
prefill_tokens_with_context = prefill_query_start_loc_cpu[
|
||||
num_prefills_with_context_cpu
|
||||
].item()
|
||||
_ChunkedMetadata = MLACommonPrefillMetadata.ChunkedContextMetadata
|
||||
if self.dcp_world_size > 1:
|
||||
chunked_context_metadata = _ChunkedMetadata(
|
||||
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
||||
starts=local_chunk_starts.to(device, non_blocking=True),
|
||||
seq_tot=padded_local_chunk_seq_lens.sum(dim=1).tolist(),
|
||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||
seq_lens=chunk_seq_lens,
|
||||
token_to_seq=token_to_seq_tensor_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
chunk_total_token=chunk_total_token.tolist(),
|
||||
workspace=self.chunked_prefill_workspace,
|
||||
padded_local_chunk_seq_lens=padded_local_chunk_seq_lens.tolist(),
|
||||
local_context_lens_allranks=local_context_lens_allranks.tolist(),
|
||||
padded_local_cu_seq_lens=padded_local_cu_chunk_seq_lens_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
padded_local_token_to_seq=padded_local_token_to_seq_tensor_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
cu_seq_lens_lst=cu_seq_lens_cpu.tolist(),
|
||||
chunk_size=padded_local_max_context_chunk_across_ranks,
|
||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||
)
|
||||
else:
|
||||
chunked_context_metadata = _ChunkedMetadata(
|
||||
cu_seq_lens=cu_seq_lens_cpu.to(device, non_blocking=True),
|
||||
starts=chunk_starts.to(device, non_blocking=True),
|
||||
seq_tot=chunk_seq_lens.sum(dim=1).tolist(),
|
||||
max_seq_lens=chunk_seq_lens.max(dim=1).values.tolist(),
|
||||
seq_lens=chunk_seq_lens,
|
||||
token_to_seq=token_to_seq_tensor_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
chunk_total_token=chunk_total_token,
|
||||
workspace=self.chunked_prefill_workspace,
|
||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||
)
|
||||
|
||||
assert (
|
||||
max(chunked_context_metadata.max_seq_lens)
|
||||
<= self.chunked_prefill_workspace_size
|
||||
)
|
||||
chunked_context_metadata = build_mla_chunked_context_metadata(
|
||||
context_lens_cpu=context_lens_cpu,
|
||||
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
|
||||
num_prefills=num_prefills,
|
||||
chunked_prefill_workspace=self.chunked_prefill_workspace,
|
||||
chunked_prefill_workspace_size=self.chunked_prefill_workspace_size,
|
||||
block_size=self.page_size,
|
||||
align_chunk_to_block=True,
|
||||
device=device,
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
dcp_local_block_size=self.dcp_local_block_size,
|
||||
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||
)
|
||||
|
||||
prefill_metadata = MLACommonPrefillMetadata(
|
||||
block_table=block_table_tensor[reqs_start:, ...],
|
||||
@@ -1985,19 +2008,14 @@ def reorg_kvcache(
|
||||
return reorganized_kv_c_normed, reorganized_k_pe
|
||||
|
||||
|
||||
class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
"""
|
||||
NOTE: Please read the comment at the top of the file before trying to
|
||||
understand this class
|
||||
Shared MLA base providing dense-MHA prefill (via the selected
|
||||
MLAPrefillBackend) for both dense and sparse impls; subclasses add decode
|
||||
(``forward_mqa``).
|
||||
"""
|
||||
|
||||
def fused_output_quant_supported(self, quant_key):
|
||||
return quant_key in (
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Dynamic64Sym,
|
||||
)
|
||||
_use_flashinfer_concat_mla_k: bool
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -2005,60 +2023,25 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int,
|
||||
alibi_slopes: list[float] | None,
|
||||
sliding_window: int | None,
|
||||
kv_cache_dtype: str,
|
||||
logits_soft_cap: float | None,
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
# MLA Specific Arguments
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
qk_head_dim: int,
|
||||
v_head_dim: int,
|
||||
kv_b_proj: ColumnParallelLinear,
|
||||
# DSV3.2 MLA Specific Arguments
|
||||
indexer: object | None = None,
|
||||
q_pad_num_heads: int | None = None,
|
||||
) -> None:
|
||||
if kv_sharing_target_layer_name is not None:
|
||||
raise NotImplementedError("KV sharing is not supported for MLA")
|
||||
|
||||
self.num_heads = num_heads
|
||||
self.head_size = head_size
|
||||
self.scale = float(scale)
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.qk_head_dim = qk_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.kv_b_proj = kv_b_proj
|
||||
self.indexer = indexer
|
||||
self.q_pad_num_heads = q_pad_num_heads
|
||||
self.supports_quant_query_input = True
|
||||
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
|
||||
|
||||
# Use flashinfer's optimized concat_mla_k kernel when available.
|
||||
# The kernel is optimized for DeepSeek V3 dimensions:
|
||||
# num_heads=128, nope_dim=128, rope_dim=64
|
||||
self._use_flashinfer_concat_mla_k = (
|
||||
has_flashinfer()
|
||||
and (self.num_heads == 128)
|
||||
and (self.qk_nope_head_dim == 128)
|
||||
and (self.qk_rope_head_dim == 64)
|
||||
)
|
||||
|
||||
self.dcp_world_size: int = -1
|
||||
|
||||
self.cp_kv_cache_interleave_size: int = (
|
||||
get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size
|
||||
)
|
||||
|
||||
def _concat_k_nope_k_pe(
|
||||
self, k_nope: torch.Tensor, k_pe: torch.Tensor
|
||||
@@ -2341,7 +2324,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
|
||||
return output, output_lse
|
||||
|
||||
def forward_mha(
|
||||
def forward_mha( # type: ignore[override]
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
@@ -2428,6 +2411,83 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
output_prefill = output_prefill.flatten(start_dim=-2)
|
||||
output.copy_(output_prefill)
|
||||
|
||||
|
||||
class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
|
||||
"""
|
||||
NOTE: Please read the comment at the top of the file before trying to
|
||||
understand this class
|
||||
"""
|
||||
|
||||
def fused_output_quant_supported(self, quant_key):
|
||||
return quant_key in (
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Dynamic64Sym,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int,
|
||||
alibi_slopes: list[float] | None,
|
||||
sliding_window: int | None,
|
||||
kv_cache_dtype: str,
|
||||
logits_soft_cap: float | None,
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
# MLA Specific Arguments
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
qk_head_dim: int,
|
||||
v_head_dim: int,
|
||||
kv_b_proj: ColumnParallelLinear,
|
||||
# DSV3.2 MLA Specific Arguments
|
||||
indexer: object | None = None,
|
||||
q_pad_num_heads: int | None = None,
|
||||
) -> None:
|
||||
if kv_sharing_target_layer_name is not None:
|
||||
raise NotImplementedError("KV sharing is not supported for MLA")
|
||||
|
||||
super().__init__(
|
||||
num_heads,
|
||||
head_size,
|
||||
scale,
|
||||
num_kv_heads,
|
||||
kv_cache_dtype,
|
||||
kv_lora_rank,
|
||||
qk_nope_head_dim,
|
||||
qk_rope_head_dim,
|
||||
qk_head_dim,
|
||||
v_head_dim,
|
||||
kv_b_proj,
|
||||
)
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.indexer = indexer
|
||||
self.q_pad_num_heads = q_pad_num_heads
|
||||
self.supports_quant_query_input = True
|
||||
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
|
||||
|
||||
# Use flashinfer's optimized concat_mla_k kernel when available.
|
||||
# The kernel is optimized for DeepSeek V3 dimensions:
|
||||
# num_heads=128, nope_dim=128, rope_dim=64
|
||||
self._use_flashinfer_concat_mla_k = (
|
||||
has_flashinfer()
|
||||
and (self.num_heads == 128)
|
||||
and (self.qk_nope_head_dim == 128)
|
||||
and (self.qk_rope_head_dim == 64)
|
||||
)
|
||||
|
||||
self.dcp_world_size: int = -1
|
||||
|
||||
self.cp_kv_cache_interleave_size: int = (
|
||||
get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def forward_mqa(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared forward_mha implementation and metadata builder for sparse MLA backends."""
|
||||
|
||||
from shutil import which
|
||||
from typing import TYPE_CHECKING, Generic, TypeVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBaseImpl,
|
||||
MLACommonPrefillMetadata,
|
||||
build_mla_chunked_context_metadata,
|
||||
get_mla_dims,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.torch_utils import np_to_pinned_tensor
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionMetadata,
|
||||
AttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
T = TypeVar("T", bound=AttentionMetadata)
|
||||
|
||||
|
||||
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
metadata_cls: type[T]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: "AttentionSpec",
|
||||
layer_names: list[str],
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.model_config = vllm_config.model_config
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
self.topk_tokens: int = vllm_config.model_config.hf_config.index_topk
|
||||
self.req_id_per_token_buffer = torch.empty(
|
||||
(vllm_config.scheduler_config.max_num_batched_tokens,),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
parallel_config = vllm_config.parallel_config
|
||||
try:
|
||||
self.dcp_world_size = get_dcp_group().world_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group
|
||||
except AssertionError:
|
||||
# DCP might not be initialized in testing
|
||||
self.dcp_world_size = 1
|
||||
self.dcp_rank = 0
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
self.dcp_local_block_size = self.cp_kv_cache_interleave_size
|
||||
self.dcp_virtual_block_size = self.dcp_local_block_size * self.dcp_world_size
|
||||
|
||||
self.chunked_prefill_workspace_size = (
|
||||
self.determine_chunked_prefill_workspace_size(vllm_config)
|
||||
)
|
||||
workspace_head_size = (
|
||||
self.mla_dims.kv_lora_rank + self.mla_dims.qk_rope_head_dim
|
||||
)
|
||||
workspace_rows = self.chunked_prefill_workspace_size
|
||||
if self.dcp_world_size > 1:
|
||||
# DCP gathers each rank's local KV shard into the workspace, so it
|
||||
# needs an extra 1/DCP rows beyond the TP allocation.
|
||||
assert self.chunked_prefill_workspace_size % self.dcp_world_size == 0
|
||||
workspace_rows += self.chunked_prefill_workspace_size // self.dcp_world_size
|
||||
self.chunked_prefill_workspace = torch.empty(
|
||||
(workspace_rows, workspace_head_size),
|
||||
dtype=self.model_config.dtype,
|
||||
device=device,
|
||||
)
|
||||
layer_prefill_backend = vllm_config.compilation_config.static_forward_context[
|
||||
layer_names[0]
|
||||
].prefill_backend
|
||||
self._prefill_backend = (
|
||||
layer_prefill_backend.clone() if layer_prefill_backend is not None else None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def determine_chunked_prefill_workspace_size(vllm_config: "VllmConfig") -> int:
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
cache_config = vllm_config.cache_config
|
||||
model_config = vllm_config.model_config
|
||||
topk_tokens = model_config.hf_config.index_topk
|
||||
|
||||
workspace_size = min(
|
||||
max(
|
||||
8 * model_config.max_model_len,
|
||||
4 * scheduler_config.max_num_seqs * cache_config.block_size,
|
||||
),
|
||||
64 * 1024,
|
||||
scheduler_config.max_num_seqs * topk_tokens,
|
||||
)
|
||||
return max(
|
||||
workspace_size,
|
||||
scheduler_config.max_num_seqs * cache_config.block_size,
|
||||
)
|
||||
|
||||
def _build_req_id_per_token(
|
||||
self,
|
||||
common_attn_metadata: "CommonAttentionMetadata",
|
||||
) -> torch.Tensor:
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
starts = np.asarray(common_attn_metadata.query_start_loc_cpu, dtype=np.int32)
|
||||
seg_lengths = np.diff(starts)
|
||||
req_id_per_token = np.repeat(
|
||||
np.arange(seg_lengths.shape[0], dtype=np.int32), seg_lengths
|
||||
)
|
||||
self.req_id_per_token_buffer.fill_(0)
|
||||
self.req_id_per_token_buffer[: req_id_per_token.shape[0]].copy_(
|
||||
np_to_pinned_tensor(req_id_per_token), non_blocking=True
|
||||
)
|
||||
return self.req_id_per_token_buffer[:num_tokens]
|
||||
|
||||
def _build_chunked_context_fields(
|
||||
self,
|
||||
common_attn_metadata: "CommonAttentionMetadata",
|
||||
num_decodes: int,
|
||||
num_prefills: int,
|
||||
prefill_query_lens_cpu: torch.Tensor | None,
|
||||
) -> "MLACommonPrefillMetadata.ChunkedContextMetadata | None":
|
||||
if num_prefills == 0 or prefill_query_lens_cpu is None:
|
||||
return None
|
||||
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
context_lens_cpu = (
|
||||
seq_lens_cpu[num_decodes : num_decodes + num_prefills]
|
||||
- prefill_query_lens_cpu
|
||||
)
|
||||
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
prefill_query_start_loc_cpu = qsl_cpu[num_decodes:] - qsl_cpu[num_decodes]
|
||||
|
||||
return build_mla_chunked_context_metadata(
|
||||
context_lens_cpu=context_lens_cpu,
|
||||
prefill_query_start_loc_cpu=prefill_query_start_loc_cpu,
|
||||
num_prefills=num_prefills,
|
||||
chunked_prefill_workspace=self.chunked_prefill_workspace,
|
||||
chunked_prefill_workspace_size=self.chunked_prefill_workspace_size,
|
||||
block_size=self.kv_cache_spec.block_size,
|
||||
align_chunk_to_block=current_platform.is_cuda(),
|
||||
device=self.device,
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
dcp_local_block_size=self.dcp_local_block_size,
|
||||
dcp_virtual_block_size=self.dcp_virtual_block_size,
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: "CommonAttentionMetadata",
|
||||
fast_build: bool = False,
|
||||
) -> T:
|
||||
req_id_per_token = self._build_req_id_per_token(common_attn_metadata)
|
||||
|
||||
num_decodes, num_prefills, num_decode_tokens, _ = split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold or 1,
|
||||
)
|
||||
(
|
||||
prefill_query_start_loc,
|
||||
prefill_max_query_len,
|
||||
prefill_query_lens_cpu,
|
||||
) = self._build_prefill_fields(common_attn_metadata, num_decodes, num_prefills)
|
||||
|
||||
prefill_max_seq_len = 0
|
||||
prefill: MLACommonPrefillMetadata | None = None
|
||||
if num_prefills > 0 and self._prefill_backend is not None:
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
prefill_max_seq_len = int(
|
||||
seq_lens_cpu[num_decodes : num_decodes + num_prefills].max().item()
|
||||
)
|
||||
prefill = MLACommonPrefillMetadata(
|
||||
block_table=common_attn_metadata.block_table_tensor[num_decodes:, ...],
|
||||
query_start_loc=prefill_query_start_loc,
|
||||
max_query_len=prefill_max_query_len,
|
||||
chunked_context=self._build_chunked_context_fields(
|
||||
common_attn_metadata,
|
||||
num_decodes,
|
||||
num_prefills,
|
||||
prefill_query_lens_cpu,
|
||||
),
|
||||
q_data_type=self.model_config.dtype,
|
||||
output_dtype=self.model_config.dtype,
|
||||
prefill_backend=self._prefill_backend,
|
||||
)
|
||||
self._prefill_backend.prepare_metadata(prefill)
|
||||
|
||||
return self.metadata_cls( # type: ignore[call-arg]
|
||||
num_reqs=common_attn_metadata.num_reqs,
|
||||
max_query_len=common_attn_metadata.max_query_len,
|
||||
max_seq_len=common_attn_metadata.max_seq_len,
|
||||
num_actual_tokens=common_attn_metadata.num_actual_tokens,
|
||||
query_start_loc=common_attn_metadata.query_start_loc,
|
||||
slot_mapping=common_attn_metadata.slot_mapping,
|
||||
block_table=common_attn_metadata.block_table_tensor,
|
||||
req_id_per_token=req_id_per_token,
|
||||
seq_lens=common_attn_metadata.seq_lens,
|
||||
block_size=self.kv_cache_spec.block_size,
|
||||
topk_tokens=self.topk_tokens,
|
||||
num_decodes=num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
prefill_max_seq_len=prefill_max_seq_len,
|
||||
prefill=prefill,
|
||||
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_prefill_fields(
|
||||
common_attn_metadata: "CommonAttentionMetadata",
|
||||
num_decodes: int,
|
||||
num_prefills: int,
|
||||
) -> tuple[
|
||||
torch.Tensor | None, # prefill_query_start_loc
|
||||
int, # prefill_max_query_len
|
||||
torch.Tensor | None, # prefill_query_lens_cpu
|
||||
]:
|
||||
if num_prefills == 0:
|
||||
return None, 0, None
|
||||
|
||||
offset = common_attn_metadata.query_start_loc[num_decodes]
|
||||
prefill_query_start_loc = (
|
||||
common_attn_metadata.query_start_loc[num_decodes:] - offset
|
||||
)
|
||||
|
||||
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
prefill_qsl_cpu = qsl_cpu[num_decodes:] - qsl_cpu[num_decodes]
|
||||
prefill_query_lens = prefill_qsl_cpu[1:] - prefill_qsl_cpu[:-1]
|
||||
prefill_max_query_len = int(prefill_query_lens.max().item())
|
||||
|
||||
return (
|
||||
prefill_query_start_loc,
|
||||
prefill_max_query_len,
|
||||
prefill_query_lens,
|
||||
)
|
||||
|
||||
|
||||
class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
"""Sparse MLA base: shared dense-MHA prefill (from MLACommonBaseImpl) plus the
|
||||
sparse top-k MQA decode path. Subclasses implement forward_mqa."""
|
||||
|
||||
is_sparse = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int,
|
||||
alibi_slopes: list[float] | None,
|
||||
sliding_window: int | None,
|
||||
kv_cache_dtype: str,
|
||||
logits_soft_cap: float | None,
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
# MLA-specific
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
qk_head_dim: int,
|
||||
v_head_dim: int,
|
||||
kv_b_proj: "ColumnParallelLinear",
|
||||
indexer: object | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
q_pad_num_heads: int | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_heads,
|
||||
head_size,
|
||||
scale,
|
||||
num_kv_heads,
|
||||
kv_cache_dtype,
|
||||
kv_lora_rank,
|
||||
qk_nope_head_dim,
|
||||
qk_rope_head_dim,
|
||||
qk_head_dim,
|
||||
v_head_dim,
|
||||
kv_b_proj,
|
||||
)
|
||||
|
||||
# The indexer carries the shared buffer for normal layers and tests;
|
||||
# the explicitly-passed buffer covers backbone skip layers, whose
|
||||
# indexer is not constructed (see deepseek_v2.py).
|
||||
self.topk_indices_buffer: torch.Tensor | None = (
|
||||
indexer.topk_indices_buffer # type: ignore[attr-defined]
|
||||
if indexer is not None
|
||||
else topk_indices_buffer
|
||||
)
|
||||
|
||||
self._use_flashinfer_concat_mla_k = (
|
||||
has_flashinfer()
|
||||
and which("ninja") is not None
|
||||
and (self.num_heads == 128)
|
||||
and (self.qk_nope_head_dim == 128)
|
||||
and (self.qk_rope_head_dim == 64)
|
||||
)
|
||||
Reference in New Issue
Block a user