[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -207,7 +207,7 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass, replace
|
||||
from enum import Enum
|
||||
from math import lcm
|
||||
from typing import ClassVar, Generic, TypeVar, cast
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -284,6 +284,7 @@ from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
MLAAttentionImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.index_group import HiSparseMLAIndexGroup
|
||||
from vllm.v1.attention.backends.mla.prefill import (
|
||||
MLAPrefillBackend,
|
||||
get_mla_prefill_backend,
|
||||
@@ -309,6 +310,11 @@ from vllm.v1.kv_cache_interface import (
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_FP8_DTYPE = current_platform.fp8_dtype()
|
||||
@@ -433,6 +439,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
use_sparse: bool = False,
|
||||
indexer: object | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
sliding_window: int | None = None,
|
||||
prefill_backend_cls: type[MLAPrefillBackend] | None = None,
|
||||
@@ -538,9 +545,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# explicitly so backbone "skip" layers (indexer=None) still find it.
|
||||
if use_sparse:
|
||||
extra_impl_args["topk_indices_buffer"] = topk_indices_buffer
|
||||
if index_group_builder is not None:
|
||||
extra_impl_args["index_group_builder"] = index_group_builder
|
||||
|
||||
impl_cls = cast(type[MLAAttentionImpl], self.attn_backend.get_impl_cls())
|
||||
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an MLAAttentionImpl subclass
|
||||
impl = impl_cls(
|
||||
num_heads=self.num_heads,
|
||||
head_size=self.head_size,
|
||||
scale=self.scale,
|
||||
@@ -562,6 +571,13 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
indexer=indexer,
|
||||
**extra_impl_args,
|
||||
)
|
||||
self.impl = impl # type: ignore[assignment]
|
||||
index_group = getattr(impl, "index_group", None)
|
||||
self.hisparse_cache = (
|
||||
index_group.cache(cast(Any, impl).index_group_index)
|
||||
if isinstance(index_group, HiSparseMLAIndexGroup)
|
||||
else None
|
||||
)
|
||||
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
|
||||
self.is_amx_bmm_enabled = getattr(self.impl, "uses_amx_bmm", False)
|
||||
# MLA reads this weight directly to build W_UK/W_UV, so a backend must
|
||||
@@ -723,6 +739,61 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
return self._chunked_prefill_workspace_size
|
||||
|
||||
def update_kv_cache(
|
||||
self,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor | None,
|
||||
attn_metadata: "MLACommonMetadata | None",
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
) -> None:
|
||||
cache = self.hisparse_cache
|
||||
if slot_mapping is None or (cache is not None and cache.dummy_batch):
|
||||
return
|
||||
kv_c_normed, k_pe, slot_mapping = maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
slot_mapping,
|
||||
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
|
||||
self.use_pcp,
|
||||
)
|
||||
assert slot_mapping is not None
|
||||
if cache is not None:
|
||||
kv_cache, slot_mapping, num_rows = cache.write_target(
|
||||
kv_c_normed.shape[0], slot_mapping.numel()
|
||||
)
|
||||
kv_c_normed = kv_c_normed[:num_rows]
|
||||
k_pe = k_pe[:num_rows]
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
)
|
||||
if cache is not None:
|
||||
mirror_target = cache.mirror_write_target(kv_c_normed.shape[0])
|
||||
if mirror_target is not None:
|
||||
mirror_cache, mirror_slots = mirror_target
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
mirror_cache,
|
||||
mirror_slots,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
)
|
||||
|
||||
def prepare_kv_cache_update(
|
||||
self, attn_metadata: "MLACommonMetadata | None"
|
||||
) -> None:
|
||||
cache = self.hisparse_cache
|
||||
if cache is not None and cache.runtime.is_group_leader:
|
||||
cache.prepare_group_for_batch(attn_metadata)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -750,25 +821,18 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
f"Expected slot_mapping to be a dict, got {type(slot_mapping)}. "
|
||||
)
|
||||
layer_slot_mapping = slot_mapping.get(self.layer_name)
|
||||
kv_for_cache, kpe_for_cache, layer_slot_mapping = (
|
||||
maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
layer_slot_mapping,
|
||||
attn_metadata.num_decode_tokens
|
||||
if attn_metadata is not None
|
||||
else None,
|
||||
self.use_pcp,
|
||||
)
|
||||
)
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
kv_for_cache,
|
||||
kpe_for_cache,
|
||||
self.prepare_kv_cache_update(attn_metadata)
|
||||
self.update_kv_cache(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
self_kv_cache,
|
||||
layer_slot_mapping,
|
||||
attn_metadata,
|
||||
self.kv_cache_dtype,
|
||||
self._k_scale,
|
||||
)
|
||||
if self.hisparse_cache is not None:
|
||||
self.hisparse_cache.finish_kv_update()
|
||||
output = torch.empty(output_shape, dtype=q.dtype, device=q.device)
|
||||
self.forward_impl(
|
||||
q,
|
||||
@@ -1113,6 +1177,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
return output_padded
|
||||
|
||||
def _use_sparse_mha(self, attn_metadata: "MLACommonMetadata") -> bool:
|
||||
if self.hisparse_cache is not None:
|
||||
return False
|
||||
prefill = attn_metadata.prefill
|
||||
if prefill is None:
|
||||
return False
|
||||
@@ -1296,6 +1362,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
return MLAAttentionSpec(
|
||||
**common_kwargs,
|
||||
is_index_group_leader=self.indexer is not None,
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
|
||||
@@ -1349,19 +1416,14 @@ def unified_mla_kv_cache_update(
|
||||
attn_metadata, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(
|
||||
layer_name
|
||||
)
|
||||
attn_layer.prepare_kv_cache_update(attn_metadata)
|
||||
if layer_slot_mapping is not None:
|
||||
kv_c_normed, k_pe, layer_slot_mapping = maybe_gather_mla_latent_cache_inputs(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
layer_slot_mapping,
|
||||
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
|
||||
attn_layer.use_pcp,
|
||||
)
|
||||
attn_layer.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
attn_layer.update_kv_cache(
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_cache,
|
||||
layer_slot_mapping,
|
||||
attn_metadata,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
)
|
||||
@@ -1409,6 +1471,8 @@ def unified_mla_attention_with_output(
|
||||
del kv_cache_dummy_dep
|
||||
layer_name = _resolve_layer_name(layer_name)
|
||||
attn_metadata, layer, kv_cache, _ = get_attention_context(layer_name)
|
||||
if layer.hisparse_cache is not None:
|
||||
layer.hisparse_cache.finish_kv_update()
|
||||
layer.forward_impl(
|
||||
q,
|
||||
kv_c_normed,
|
||||
@@ -1501,6 +1565,7 @@ class MLACommonBackend(AttentionBackend):
|
||||
def customize_spec(cls, spec: "AttentionSpec") -> "AttentionSpec":
|
||||
"""Per-token-head modes pack an inline fp32 scale pair after the
|
||||
latent data (single-sided: ``head_size_v == 0`` for MLA)."""
|
||||
spec = super().customize_spec(spec)
|
||||
mode = spec.kv_quant_mode
|
||||
if spec.state_content_bytes is not None or not mode.is_per_token_head:
|
||||
return spec
|
||||
@@ -2090,7 +2155,11 @@ def build_mla_chunked_context_metadata(
|
||||
local_token_offset += num_local_tokens
|
||||
|
||||
seq_lens_cpu = _flat_int32(seq_lens_flat)
|
||||
starts = _flat_int32(starts_flat).to(device, non_blocking=True)
|
||||
starts_and_context_lens = _flat_int32(starts_flat + context_lens).to(
|
||||
device, non_blocking=True
|
||||
)
|
||||
starts = starts_and_context_lens[: len(starts_flat)]
|
||||
context_lens_gpu = starts_and_context_lens[len(starts_flat) :]
|
||||
cu_seq_lens = _flat_int32(cu_seq_lens_flat).to(device, non_blocking=True)
|
||||
cu_seqlens_q = _flat_int32(cu_seqlens_q_flat).to(device, non_blocking=True)
|
||||
token_to_seq = _flat_int32(np.concatenate(token_to_seq_parts)).to(
|
||||
@@ -2145,7 +2214,7 @@ def build_mla_chunked_context_metadata(
|
||||
chunks.append(chunk)
|
||||
|
||||
return MLACommonPrefillMetadata.ChunkedContextMetadata(
|
||||
context_lens=context_lens_cpu.to(device, non_blocking=True),
|
||||
context_lens=context_lens_gpu,
|
||||
workspace=chunked_prefill_workspace,
|
||||
chunks=chunks,
|
||||
context_lens_list=context_lens,
|
||||
@@ -2835,12 +2904,12 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
output_lse = None
|
||||
for chunk in chunked_context.chunks:
|
||||
toks = chunk.num_context_tokens
|
||||
block_table = prefill_metadata.block_table[chunk.request_slice]
|
||||
chunk_block_table = prefill_metadata.block_table[chunk.request_slice]
|
||||
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=block_table,
|
||||
block_table=chunk_block_table,
|
||||
workspace_starts=chunk.cu_seq_lens,
|
||||
batch_size=chunk.num_requests,
|
||||
seq_starts=chunk.starts,
|
||||
@@ -2853,7 +2922,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
ops.gather_mla_context_cache_cpu(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=block_table,
|
||||
block_table=chunk_block_table,
|
||||
starts=chunk.starts,
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
)
|
||||
@@ -2861,7 +2930,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=block_table,
|
||||
block_table=chunk_block_table,
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
token_to_seq=chunk.token_to_seq,
|
||||
num_tokens=toks,
|
||||
@@ -2874,7 +2943,7 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
ops.cp_gather_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=block_table,
|
||||
block_table=chunk_block_table,
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
batch_size=chunk.num_requests,
|
||||
seq_starts=chunk.starts,
|
||||
|
||||
@@ -3,13 +3,15 @@
|
||||
"""Shared MHA implementation and metadata builder for sparse MLA backends."""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from shutil import which
|
||||
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar, cast
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_dcp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -17,6 +19,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBaseImpl,
|
||||
MLACommonDecodeMetadata,
|
||||
MLACommonMetadata,
|
||||
MLACommonPrefillMetadata,
|
||||
accumulate_mla_context_chunk,
|
||||
@@ -33,11 +36,19 @@ from vllm.utils.torch_utils import (
|
||||
is_quantized_kv_cache,
|
||||
np_to_pinned_tensor,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata, AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backend import AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroup,
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparsePrefillStagingPlan,
|
||||
build_hisparse_prefill_staging_plan,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -47,8 +58,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
T = TypeVar("T", bound=AttentionMetadata)
|
||||
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES = 128 * 1024 * 1024 # 128 MiB
|
||||
|
||||
|
||||
@@ -116,9 +125,45 @@ def _is_masked_mha_available(
|
||||
return fa_version == 4 and not is_quantized_kv_cache(kv_cache_dtype)
|
||||
|
||||
|
||||
def _use_dense_mha_prefill(
|
||||
vllm_config: "VllmConfig", prefill_max_seq_len: int, topk_tokens: int
|
||||
) -> bool:
|
||||
attention_config = vllm_config.attention_config
|
||||
return (
|
||||
attention_config.hisparse_config is None
|
||||
and not attention_config.sparse_mla_force_mqa
|
||||
and prefill_max_seq_len <= topk_tokens
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseMLAPrefillMetadata(MLACommonPrefillMetadata):
|
||||
host_staging_plan: HiSparsePrefillStagingPlan | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class SparseMLACommonMetadata(MLACommonMetadata[MLACommonDecodeMetadata]):
|
||||
block_table: torch.Tensor
|
||||
req_id_per_token: torch.Tensor
|
||||
seq_lens: torch.Tensor | None = None
|
||||
block_size: int = 64
|
||||
topk_tokens: int = 2048
|
||||
num_decodes: int = 0
|
||||
num_prefills: int = 0
|
||||
num_decode_tokens: int = 0
|
||||
decode_max_query_len: int = 0
|
||||
prefill_max_seq_len: int = 0
|
||||
prefill: SparseMLAPrefillMetadata | None = None
|
||||
cp_kv_cache_interleave_size: int = 1
|
||||
|
||||
|
||||
T = TypeVar("T", bound=SparseMLACommonMetadata)
|
||||
|
||||
|
||||
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
metadata_cls: type[T]
|
||||
require_uniform_decodes: ClassVar[bool] = False
|
||||
hisparse_supports_multi_token_decode: ClassVar[bool] = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -138,6 +183,12 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self._context_lens_cpu = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_seqs,
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
try:
|
||||
@@ -197,6 +248,22 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
layer_prefill_backend.clone() if layer_prefill_backend is not None else None
|
||||
)
|
||||
|
||||
def _init_reorder_batch_threshold(
|
||||
self,
|
||||
reorder_batch_threshold: int | None = 1,
|
||||
supports_spec_as_decode: bool = False,
|
||||
supports_dcp_with_varlen: bool = False,
|
||||
) -> None:
|
||||
if self.vllm_config.attention_config.hisparse_config is not None:
|
||||
reorder_batch_threshold = 1
|
||||
if not self.hisparse_supports_multi_token_decode:
|
||||
supports_spec_as_decode = False
|
||||
super()._init_reorder_batch_threshold(
|
||||
reorder_batch_threshold,
|
||||
supports_spec_as_decode,
|
||||
supports_dcp_with_varlen,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def determine_chunked_prefill_workspace_size(vllm_config: "VllmConfig") -> int:
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
@@ -245,9 +312,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
context_lens_cpu = torch.empty(
|
||||
num_prefills, dtype=seq_lens_cpu.dtype, pin_memory=PIN_MEMORY
|
||||
)
|
||||
context_lens_cpu = self._context_lens_cpu[:num_prefills]
|
||||
torch.subtract(
|
||||
seq_lens_cpu[num_decodes : num_decodes + num_prefills],
|
||||
prefill_query_lens_cpu,
|
||||
@@ -289,34 +354,66 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
prefill_max_query_len,
|
||||
prefill_query_lens_cpu,
|
||||
) = self._build_prefill_fields(common_attn_metadata, num_decodes, num_prefills)
|
||||
decode_max_query_len = 0
|
||||
if num_decodes:
|
||||
if common_attn_metadata.max_logits_per_req is not None:
|
||||
decode_max_query_len = common_attn_metadata.max_logits_per_req
|
||||
elif num_prefills == 0:
|
||||
decode_max_query_len = common_attn_metadata.max_query_len
|
||||
else:
|
||||
decode_query_lens_cpu = torch.diff(
|
||||
common_attn_metadata.query_start_loc_cpu[: num_decodes + 1]
|
||||
)
|
||||
decode_max_query_len = int(decode_query_lens_cpu.max().item())
|
||||
|
||||
prefill_max_seq_len = 0
|
||||
prefill: MLACommonPrefillMetadata | None = None
|
||||
prefill: SparseMLAPrefillMetadata | None = None
|
||||
if num_prefills > 0 and self._prefill_backend is not None:
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
prefill_max_seq_len = int(
|
||||
seq_lens_cpu[num_decodes : num_decodes + num_prefills].max().item()
|
||||
)
|
||||
prefill = MLACommonPrefillMetadata(
|
||||
block_table=common_attn_metadata.block_table_tensor[num_decodes:, ...],
|
||||
block_table = common_attn_metadata.block_table_tensor[num_decodes:, ...]
|
||||
chunked_context = self._build_chunked_context_fields(
|
||||
common_attn_metadata,
|
||||
num_decodes,
|
||||
num_prefills,
|
||||
prefill_query_lens_cpu,
|
||||
)
|
||||
staging_plan = None
|
||||
if self.vllm_config.attention_config.hisparse_config is not None:
|
||||
prefill_seq_lens_cpu = seq_lens_cpu[
|
||||
num_decodes : num_decodes + num_prefills
|
||||
]
|
||||
staging_block_capacity = int(
|
||||
(
|
||||
(prefill_seq_lens_cpu + self.kv_cache_spec.block_size - 1)
|
||||
// self.kv_cache_spec.block_size
|
||||
).sum()
|
||||
)
|
||||
staging_plan = build_hisparse_prefill_staging_plan(
|
||||
block_table,
|
||||
common_attn_metadata.seq_lens[num_decodes:],
|
||||
self.kv_cache_spec.block_size,
|
||||
staging_block_capacity,
|
||||
)
|
||||
if chunked_context is not None:
|
||||
block_table = staging_plan.block_table
|
||||
prefill = SparseMLAPrefillMetadata(
|
||||
block_table=block_table,
|
||||
query_start_loc=prefill_query_start_loc,
|
||||
max_query_len=prefill_max_query_len,
|
||||
query_lens_cpu=prefill_query_lens_cpu,
|
||||
chunked_context=self._build_chunked_context_fields(
|
||||
common_attn_metadata,
|
||||
num_decodes,
|
||||
num_prefills,
|
||||
prefill_query_lens_cpu,
|
||||
),
|
||||
chunked_context=chunked_context,
|
||||
q_data_type=self.model_config.dtype,
|
||||
output_dtype=self.model_config.dtype,
|
||||
prefill_backend=self._prefill_backend,
|
||||
use_dense_mha=(
|
||||
prefill_max_seq_len <= self.topk_tokens
|
||||
and not self.vllm_config.attention_config.sparse_mla_force_mqa
|
||||
use_dense_mha=_use_dense_mha_prefill(
|
||||
self.vllm_config, prefill_max_seq_len, self.topk_tokens
|
||||
),
|
||||
topk_mask_workspace=self.topk_mask_workspace,
|
||||
host_staging_plan=staging_plan,
|
||||
)
|
||||
self._prefill_backend.prepare_metadata(prefill)
|
||||
|
||||
@@ -335,6 +432,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
num_decodes=num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
decode_max_query_len=decode_max_query_len,
|
||||
prefill_max_seq_len=prefill_max_seq_len,
|
||||
prefill=prefill,
|
||||
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
|
||||
@@ -560,6 +658,7 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
kv_b_proj: "ColumnParallelLinear",
|
||||
indexer: object | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
|
||||
q_pad_num_heads: int | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
@@ -577,6 +676,21 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
)
|
||||
|
||||
self.init_topk_indices_buffer(indexer, topk_indices_buffer)
|
||||
self.index_group: SparseMLAIndexGroup | None = None
|
||||
self.index_group_index = 0
|
||||
if index_group_builder is None and self.topk_indices_buffer is not None:
|
||||
index_group_builder = SparseMLAIndexGroupBuilder(self.topk_indices_buffer)
|
||||
if index_group_builder is not None:
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.index_group, self.index_group_index = (
|
||||
index_group_builder.register_layer(
|
||||
indexer is not None,
|
||||
vllm_config,
|
||||
head_size=head_size,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
)
|
||||
)
|
||||
|
||||
self._use_flashinfer_concat_mla_k = (
|
||||
has_flashinfer()
|
||||
and which("ninja") is not None
|
||||
@@ -593,6 +707,33 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
)
|
||||
|
||||
def record_logical_topk_ready(self) -> None:
|
||||
if self.index_group is not None:
|
||||
self.index_group.set_logical_topk_ready(self.index_group_index)
|
||||
|
||||
def prepare_for_batch(self, attn_metadata: T | None) -> None:
|
||||
if self.index_group is not None:
|
||||
self.index_group.prepare_for_batch(self.index_group_index, attn_metadata)
|
||||
|
||||
def _convert_logical_to_physical_topk(
|
||||
self,
|
||||
logical_topk_indices: torch.Tensor,
|
||||
attn_metadata: Any,
|
||||
*,
|
||||
block_stride_rows: int | None,
|
||||
return_valid_counts: bool,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
group = self.index_group
|
||||
assert group is not None
|
||||
assert self.dcp_world_size == 1
|
||||
return group.convert_logical_to_physical_topk(
|
||||
self.index_group_index,
|
||||
logical_topk_indices,
|
||||
attn_metadata,
|
||||
block_stride_rows=block_stride_rows,
|
||||
return_valid_counts=return_valid_counts,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def masked_mha_workspace_fits(prefill: MLACommonPrefillMetadata) -> bool:
|
||||
"""Whether this prefill batch's top-k masks fit the workspace."""
|
||||
@@ -853,8 +994,10 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
output: torch.Tensor,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
|
||||
topk_tokens = attn_metadata.topk_tokens # type: ignore[attr-defined]
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
assert prefill_metadata is not None
|
||||
prefill_max_seq_len = attn_metadata.prefill_max_seq_len
|
||||
topk_tokens = attn_metadata.topk_tokens
|
||||
force_dense = getattr(self, "_sparse_mla_force_dense_mha", False)
|
||||
force_masked = getattr(self, "_sparse_mla_force_masked_mha", False)
|
||||
if force_dense or (prefill_max_seq_len <= topk_tokens and not force_masked):
|
||||
@@ -863,7 +1006,7 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_c_and_k_pe_cache,
|
||||
cast(MLACommonMetadata, attn_metadata),
|
||||
attn_metadata,
|
||||
k_scale,
|
||||
output,
|
||||
output_scale,
|
||||
@@ -871,13 +1014,11 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
|
||||
assert output_scale is None
|
||||
assert self.masked_mha_available
|
||||
prefill_metadata = attn_metadata.prefill # type: ignore[attr-defined]
|
||||
assert prefill_metadata is not None
|
||||
assert prefill_metadata.query_lens_cpu is not None
|
||||
assert self.topk_indices_buffer is not None
|
||||
|
||||
q_lens = prefill_metadata.query_lens_cpu.tolist()
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens # type: ignore[attr-defined]
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
topk_all = self.topk_indices_buffer[
|
||||
num_decode_tokens : num_decode_tokens + q.shape[0]
|
||||
]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
@@ -11,6 +12,11 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MLAModules:
|
||||
@@ -30,6 +36,7 @@ class MLAModules:
|
||||
topk_indices_buffer: torch.Tensor | None
|
||||
indexer_rotary_emb: torch.nn.Module | None = None
|
||||
g_proj: torch.nn.Module | None = None
|
||||
index_group_builder: "SparseMLAIndexGroupBuilder | None" = None
|
||||
|
||||
|
||||
# --8<-- [start:multi_head_latent_attention]
|
||||
@@ -128,6 +135,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
use_sparse=self.is_sparse,
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
index_group_builder=mla_modules.index_group_builder,
|
||||
non_causal_multi_token_decode=non_causal_multi_token_decode,
|
||||
)
|
||||
indexer_op = getattr(self.indexer, "indexer_op", None)
|
||||
@@ -223,6 +231,8 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
|
||||
if self.indexer and self.is_sparse and not self.skip_topk:
|
||||
self.indexer(hidden_states, q_c, positions, self.indexer_rope_emb)
|
||||
if self.is_sparse:
|
||||
self.mla_attn.impl.record_logical_topk_ready() # type: ignore[attr-defined]
|
||||
|
||||
if llama_4_scaling is not None:
|
||||
q *= llama_4_scaling
|
||||
|
||||
@@ -425,7 +425,6 @@ def sparse_attn_indexer(
|
||||
quant_block_size,
|
||||
scale_fmt,
|
||||
)
|
||||
|
||||
# The indexer and main MLA may classify the same short extend differently
|
||||
# because they use independent decode thresholds. Only the main MLA route
|
||||
# can determine whether the top-k indices will be consumed.
|
||||
@@ -955,6 +954,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.dcp_rank,
|
||||
self.dcp_world_size,
|
||||
self.cp_kv_cache_interleave_size,
|
||||
False,
|
||||
candidate_blocks=self.candidate_blocks,
|
||||
candidate_block_size=self.candidate_block_size,
|
||||
candidate_write=self.candidate_write,
|
||||
|
||||
@@ -103,10 +103,14 @@ from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionType
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
get_sparse_mla_index_group_max_rows,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerBackend,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MLAAttentionSpec, SparseCacheRole
|
||||
|
||||
from .interfaces import (
|
||||
MixtureOfExperts,
|
||||
@@ -656,6 +660,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.dtype,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
) # Only has one vector instead of K + V
|
||||
|
||||
def forward(self): ...
|
||||
@@ -1012,6 +1017,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
|
||||
input_size: int | None = None,
|
||||
reduce_results: bool = True,
|
||||
non_causal_multi_token_decode: bool = False,
|
||||
@@ -1199,6 +1205,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
indexer_rotary_emb=self.indexer_rope_emb,
|
||||
is_sparse=self.is_v32,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
)
|
||||
|
||||
self.mla_attn = MultiHeadLatentAttentionWrapper(
|
||||
@@ -1242,6 +1249,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
prefix: str,
|
||||
config: DeepseekV2Config | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -1290,6 +1298,11 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
and parallel_config.pipeline_parallel_size == 1
|
||||
and is_moe_layer
|
||||
)
|
||||
attn_kwargs = (
|
||||
{"index_group_builder": index_group_builder}
|
||||
if attn_cls is DeepseekV2MLAAttention
|
||||
else {}
|
||||
)
|
||||
self.self_attn = attn_cls(
|
||||
vllm_config=vllm_config,
|
||||
config=config,
|
||||
@@ -1306,6 +1319,7 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.self_attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
reduce_results=not self.use_sequence_parallel_moe,
|
||||
**attn_kwargs,
|
||||
)
|
||||
|
||||
if is_moe_layer:
|
||||
@@ -1429,6 +1443,14 @@ class DeepseekV2Model(nn.Module):
|
||||
)
|
||||
else:
|
||||
topk_indices_buffer = None
|
||||
index_group_builder = (
|
||||
SparseMLAIndexGroupBuilder(
|
||||
topk_indices_buffer,
|
||||
get_sparse_mla_index_group_max_rows(vllm_config),
|
||||
)
|
||||
if topk_indices_buffer is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
@@ -1445,6 +1467,7 @@ class DeepseekV2Model(nn.Module):
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
@@ -67,6 +67,10 @@ from vllm.model_executor.models.glm4_moe import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
get_sparse_mla_index_group_max_rows,
|
||||
)
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
@@ -108,6 +112,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
prefix: str,
|
||||
config: "Glm4MoeLiteConfig | None" = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
index_group_builder: SparseMLAIndexGroupBuilder | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -136,6 +141,11 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
attn_cls = Glm4MoeLiteMLAAttention
|
||||
else:
|
||||
attn_cls = Glm4MoeLiteAttention
|
||||
attn_kwargs: dict[str, typing.Any] = (
|
||||
{"index_group_builder": index_group_builder}
|
||||
if attn_cls is Glm4MoeLiteMLAAttention
|
||||
else {}
|
||||
)
|
||||
|
||||
self.self_attn = attn_cls(
|
||||
vllm_config=vllm_config,
|
||||
@@ -152,6 +162,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
**attn_kwargs,
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -234,6 +245,14 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
)
|
||||
else:
|
||||
topk_indices_buffer = None
|
||||
index_group_builder = (
|
||||
SparseMLAIndexGroupBuilder(
|
||||
topk_indices_buffer,
|
||||
get_sparse_mla_index_group_max_rows(vllm_config),
|
||||
)
|
||||
if topk_indices_buffer is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
@@ -252,6 +271,7 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
config=config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
@@ -51,6 +51,10 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
SparseMLAIndexGroupBuilder,
|
||||
get_sparse_mla_index_group_max_rows,
|
||||
)
|
||||
|
||||
from .glm4_moe_lite import (
|
||||
Glm4MixtureOfExperts,
|
||||
@@ -108,6 +112,14 @@ class Glm4MoeLiteMultiTokenPredictorLayer(nn.Module):
|
||||
)
|
||||
else:
|
||||
topk_indices_buffer = None
|
||||
index_group_builder = (
|
||||
SparseMLAIndexGroupBuilder(
|
||||
topk_indices_buffer,
|
||||
get_sparse_mla_index_group_max_rows(vllm_config),
|
||||
)
|
||||
if topk_indices_buffer is not None
|
||||
else None
|
||||
)
|
||||
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
@@ -117,6 +129,7 @@ class Glm4MoeLiteMultiTokenPredictorLayer(nn.Module):
|
||||
prefix=prefix,
|
||||
config=self.config,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
index_group_builder=index_group_builder,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -47,6 +47,35 @@ _FLASHINFER_SM120_SPARSE_MLA_DECODE_LABELS = {
|
||||
_SPARSE_MLA_MIXED_WARMUP_TOKENS = 16
|
||||
|
||||
|
||||
def autotune_hisparse_flashinfer_attention(runner: "GPUModelRunner") -> None:
|
||||
"""Autotune each HiSparse FlashInfer sparse-MLA configuration."""
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseImpl,
|
||||
)
|
||||
|
||||
tuned: set[tuple[object, ...]] = set()
|
||||
for layer in runner.vllm_config.compilation_config.static_forward_context.values():
|
||||
impl = getattr(layer, "impl", None)
|
||||
if not isinstance(impl, FlashInferMLASparseImpl):
|
||||
continue
|
||||
if getattr(layer, "hisparse_cache", None) is None:
|
||||
continue
|
||||
if impl.topk_indices_buffer is None:
|
||||
continue
|
||||
key = (
|
||||
impl.kv_cache_dtype,
|
||||
impl.num_heads,
|
||||
impl.qk_nope_head_dim,
|
||||
impl.qk_rope_head_dim,
|
||||
impl.kv_lora_rank,
|
||||
impl.topk_indices_buffer.shape[1],
|
||||
)
|
||||
if key in tuned:
|
||||
continue
|
||||
impl.autotune_hisparse_decode(layer)
|
||||
tuned.add(key)
|
||||
|
||||
|
||||
def _attention_backend_name(backend: object) -> str | None:
|
||||
get_name = getattr(backend, "get_name", None)
|
||||
if get_name is None:
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.warmup.flashinfer_autotune_cache import (
|
||||
write_flashinfer_autotune_cache,
|
||||
)
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
deepseek_v4_sparse_mla_attention_warmup,
|
||||
flashinfer_sparse_mla_decode_autotune_warmup,
|
||||
)
|
||||
@@ -353,9 +354,12 @@ def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ..
|
||||
return tuple(dict.fromkeys(token_counts))
|
||||
|
||||
|
||||
def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
|
||||
def _run_flashinfer_autotune_dummy_runs(
|
||||
runner: "GPUModelRunner", *, skip_attn: bool = False
|
||||
) -> None:
|
||||
import vllm.utils.flashinfer as fi_utils
|
||||
|
||||
dummy_run_kwargs = {"skip_attn": True} if skip_attn else {}
|
||||
for num_tokens in _flashinfer_autotune_token_counts(runner):
|
||||
tuning_buckets = fi_utils.flashinfer_get_hybrid_num_tokens_buckets(num_tokens)
|
||||
logger.info(
|
||||
@@ -369,6 +373,7 @@ def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
**dummy_run_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -431,7 +436,14 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
torch.inference_mode(),
|
||||
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
|
||||
):
|
||||
_run_flashinfer_autotune_dummy_runs(runner)
|
||||
hisparse_enabled = (
|
||||
runner.vllm_config.attention_config.hisparse_config is not None
|
||||
)
|
||||
if hisparse_enabled:
|
||||
# HiSparse hot-buffer attention is bounded by decode batch
|
||||
# size, not the prefill-sized batch used for the full model.
|
||||
autotune_hisparse_flashinfer_attention(runner)
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=hisparse_enabled)
|
||||
replayssm_autotune_warmup(runner)
|
||||
_autotune_kimi_k3_kda_qkvg(runner.get_model())
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user