[Feature] Support DCP with FP8 KV cache in MLA decode path (#44044)
Signed-off-by: shivampr <shivampr.dev@gmail.com> Signed-off-by: Shivam <shivamprasad91@gmail.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Matthew Bonanni
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d511b5bae9
commit
49f2104c53
@@ -1015,6 +1015,78 @@ def test_gather_and_maybe_dequant_cache_mla(
|
||||
torch.testing.assert_close(dst, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lora_rank", [512])
|
||||
@pytest.mark.parametrize("qk_rope_head_dim", [64])
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
@pytest.mark.parametrize("num_blocks", [128])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_gather_and_maybe_dequant_cache_mla_with_seq_starts(
|
||||
kv_lora_rank,
|
||||
qk_rope_head_dim,
|
||||
block_size,
|
||||
num_blocks,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
device,
|
||||
):
|
||||
entry_size = kv_lora_rank + qk_rope_head_dim
|
||||
scale = torch.tensor(0.1, dtype=torch.float32, device=device)
|
||||
src_cache = _create_mla_cache(
|
||||
num_blocks, block_size, entry_size, dtype, kv_cache_dtype, device
|
||||
)
|
||||
_fill_mla_cache(src_cache, kv_cache_dtype=kv_cache_dtype)
|
||||
|
||||
seq_starts = torch.tensor([3, 17, 5], dtype=torch.int32, device=device)
|
||||
seq_lens = torch.tensor([20, 10, 16], dtype=torch.int32, device=device)
|
||||
batch_size = seq_lens.shape[0]
|
||||
total_tokens = seq_lens.sum().item()
|
||||
cu_seq_lens = torch.empty((batch_size + 1), dtype=torch.int32, device=device)
|
||||
cu_seq_lens[0] = 0
|
||||
cu_seq_lens[1:] = seq_lens.cumsum(dim=0)
|
||||
token_to_seq = torch.repeat_interleave(
|
||||
torch.arange(batch_size, dtype=torch.int32, device=device), seq_lens
|
||||
)
|
||||
|
||||
block_table = torch.empty(
|
||||
(batch_size, num_blocks), dtype=torch.int32, device=device
|
||||
)
|
||||
for b in range(batch_size):
|
||||
block_table[b, :] = torch.randperm(num_blocks, device=device)
|
||||
|
||||
if kv_cache_dtype == "fp8":
|
||||
dequant_src_cache = torch.empty_like(src_cache, dtype=dtype)
|
||||
ops.convert_fp8(dequant_src_cache, src_cache, scale.item())
|
||||
else:
|
||||
dequant_src_cache = src_cache
|
||||
|
||||
expected_rows = []
|
||||
for b in range(batch_size):
|
||||
start = seq_starts[b].item()
|
||||
length = seq_lens[b].item()
|
||||
for offset in range(start, start + length):
|
||||
block_id = block_table[b, offset // block_size]
|
||||
slot = offset % block_size
|
||||
expected_rows.append(dequant_src_cache[block_id, slot])
|
||||
expected = torch.stack(expected_rows)
|
||||
|
||||
dst = torch.zeros((total_tokens, entry_size), dtype=dtype, device=device)
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache,
|
||||
dst,
|
||||
block_table,
|
||||
cu_seq_lens,
|
||||
token_to_seq,
|
||||
total_tokens,
|
||||
kv_cache_dtype,
|
||||
scale,
|
||||
seq_starts,
|
||||
)
|
||||
torch.testing.assert_close(dst, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_lora_rank", [512])
|
||||
@pytest.mark.parametrize("qk_rope_head_dim", [64])
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
|
||||
@@ -19,6 +19,7 @@ from tests.v1.attention.utils import (
|
||||
)
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention import mla_attention as mla_attention_module
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLAAttention,
|
||||
QueryLenSupport,
|
||||
@@ -30,6 +31,7 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
|
||||
from vllm.v1.attention.backends.mla import flashmla as flashmla_module
|
||||
from vllm.v1.attention.backends.mla.prefill import (
|
||||
MLAPrefillBackendEnum,
|
||||
get_mla_prefill_backend,
|
||||
@@ -552,6 +554,10 @@ class MockMLAAttentionLayer(MLAAttention):
|
||||
)
|
||||
else:
|
||||
mqa_q = (mqa_ql_nope, mqa_q_pe)
|
||||
if self.impl.dcp_world_size > 1:
|
||||
if isinstance(mqa_q, tuple):
|
||||
mqa_q = torch.cat(mqa_q, dim=-1)
|
||||
mqa_q = mla_attention_module.get_dcp_group().all_gather(mqa_q, dim=1)
|
||||
|
||||
attn_out, _ = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self)
|
||||
|
||||
@@ -569,6 +575,215 @@ class MockMLAAttentionLayer(MLAAttention):
|
||||
return output
|
||||
|
||||
|
||||
def test_mock_mla_dcp_fp8_decode_gathers_quantized_query(
|
||||
monkeypatch, default_vllm_config
|
||||
):
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA is required for FP8 decode query quantization path.")
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
num_tokens = 2
|
||||
num_heads = 2
|
||||
qk_nope_head_dim = 4
|
||||
qk_rope_head_dim = 2
|
||||
v_head_dim = 3
|
||||
kv_lora_rank = 5
|
||||
|
||||
class _DummyKVProj:
|
||||
def __init__(self):
|
||||
# Shape expected by MockMLAAttentionLayer.__init__
|
||||
self.weight = torch.randn(
|
||||
num_heads * (qk_nope_head_dim + v_head_dim),
|
||||
kv_lora_rank,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
class _FakeImpl:
|
||||
def __init__(self):
|
||||
self.kv_cache_dtype = "fp8"
|
||||
self.supports_quant_query_input = True
|
||||
self.dcp_world_size = 2
|
||||
self.forward_q = None
|
||||
|
||||
def forward_mha(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
def forward_mqa(self, q, kv_cache, attn_metadata, layer):
|
||||
self.forward_q = q
|
||||
assert isinstance(q, torch.Tensor)
|
||||
bsz, _, _ = q.shape
|
||||
return (
|
||||
torch.zeros(
|
||||
bsz,
|
||||
num_heads,
|
||||
kv_lora_rank,
|
||||
device=q.device,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
class _FakeDCPGroup:
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
self.input_dtype = None
|
||||
self.input_shape = None
|
||||
|
||||
def all_gather(self, x, dim=1):
|
||||
self.calls += 1
|
||||
self.input_dtype = x.dtype
|
||||
self.input_shape = tuple(x.shape)
|
||||
return torch.cat([x, x], dim=dim)
|
||||
|
||||
fake_group = _FakeDCPGroup()
|
||||
monkeypatch.setattr(mla_attention_module, "get_dcp_group", lambda: fake_group)
|
||||
|
||||
impl = _FakeImpl()
|
||||
with set_current_vllm_config(default_vllm_config):
|
||||
layer = MockMLAAttentionLayer(
|
||||
impl=impl,
|
||||
num_heads=num_heads,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
device=device,
|
||||
kv_b_proj=_DummyKVProj(),
|
||||
q_scale=1.0,
|
||||
k_scale=1.0,
|
||||
)
|
||||
|
||||
q = torch.randn(
|
||||
num_tokens,
|
||||
num_heads,
|
||||
qk_nope_head_dim + qk_rope_head_dim,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
kv_c = torch.randn(num_tokens, kv_lora_rank, device=device, dtype=torch.float32)
|
||||
k_pe = torch.randn(
|
||||
num_tokens, 1, qk_rope_head_dim, device=device, dtype=torch.float32
|
||||
)
|
||||
kv_cache = torch.empty(0, device=device, dtype=torch.float32)
|
||||
output = torch.empty(
|
||||
num_tokens, num_heads * v_head_dim, device=device, dtype=torch.float32
|
||||
)
|
||||
|
||||
class _AttnMeta:
|
||||
num_decode_tokens = num_tokens
|
||||
num_decodes = 1
|
||||
num_prefills = 0
|
||||
slot_mapping = torch.empty(0, dtype=torch.long, device=device)
|
||||
|
||||
layer.forward_impl(q, kv_c, k_pe, kv_cache, _AttnMeta(), output)
|
||||
|
||||
assert fake_group.calls == 1
|
||||
assert fake_group.input_dtype == current_platform.fp8_dtype()
|
||||
assert fake_group.input_shape == (
|
||||
num_tokens,
|
||||
num_heads,
|
||||
kv_lora_rank + qk_rope_head_dim,
|
||||
)
|
||||
assert isinstance(impl.forward_q, torch.Tensor)
|
||||
assert tuple(impl.forward_q.shape) == (
|
||||
num_tokens,
|
||||
num_heads * impl.dcp_world_size,
|
||||
kv_lora_rank + qk_rope_head_dim,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_fp8_kvcache", [False, True], ids=["bf16", "fp8"])
|
||||
def test_flashmla_dcp_decode_metadata_uses_gathered_query_heads(
|
||||
monkeypatch, is_fp8_kvcache
|
||||
):
|
||||
class _FakeSchedulerMetadata:
|
||||
tile_scheduler_metadata = None
|
||||
num_splits = None
|
||||
|
||||
base_call: tuple[torch.Tensor, int, int, bool] | None = None
|
||||
fp8_call: tuple[torch.Tensor, int, int] | None = None
|
||||
|
||||
def fake_get_mla_metadata(
|
||||
seq_lens_device,
|
||||
num_q_tokens_per_head_k,
|
||||
num_heads_k,
|
||||
is_fp8_kvcache=False,
|
||||
):
|
||||
nonlocal base_call
|
||||
base_call = (
|
||||
seq_lens_device,
|
||||
num_q_tokens_per_head_k,
|
||||
num_heads_k,
|
||||
is_fp8_kvcache,
|
||||
)
|
||||
return _FakeSchedulerMetadata(), None
|
||||
|
||||
def fake_get_mla_metadata_dense_fp8(
|
||||
seq_lens_device, num_q_tokens_per_head_k, num_heads_k
|
||||
):
|
||||
nonlocal fp8_call
|
||||
fp8_call = (
|
||||
seq_lens_device,
|
||||
num_q_tokens_per_head_k,
|
||||
num_heads_k,
|
||||
)
|
||||
return (
|
||||
torch.empty((0, 8), dtype=torch.int32),
|
||||
torch.empty((0,), dtype=torch.int32),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(flashmla_module, "get_mla_metadata", fake_get_mla_metadata)
|
||||
monkeypatch.setattr(
|
||||
flashmla_module,
|
||||
"get_mla_metadata_dense_fp8",
|
||||
fake_get_mla_metadata_dense_fp8,
|
||||
)
|
||||
|
||||
builder = object.__new__(flashmla_module.FlashMLAMetadataBuilder)
|
||||
builder.num_q_heads = 4
|
||||
builder.dcp_world_size = 2
|
||||
builder.is_fp8_kvcache = is_fp8_kvcache
|
||||
builder.compilation_config = type(
|
||||
"_CompilationConfig",
|
||||
(),
|
||||
{
|
||||
"cudagraph_mode": type(
|
||||
"_CudaGraphMode",
|
||||
(),
|
||||
{"has_full_cudagraphs": lambda self: False},
|
||||
)()
|
||||
},
|
||||
)()
|
||||
|
||||
seq_lens = torch.tensor([16, 24], dtype=torch.int32)
|
||||
query_start_loc = torch.tensor([0, 1, 2], dtype=torch.int32)
|
||||
|
||||
metadata = builder._build_decode(
|
||||
block_table_tensor=torch.empty((2, 1), dtype=torch.int32),
|
||||
seq_lens_device=seq_lens,
|
||||
max_seq_len=24,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
query_start_loc_device=query_start_loc,
|
||||
num_decode_tokens=2,
|
||||
dcp_tot_seq_lens_device=None,
|
||||
)
|
||||
|
||||
assert base_call is not None
|
||||
assert base_call[0] is seq_lens
|
||||
assert base_call[1:] == (8, 1, is_fp8_kvcache)
|
||||
if is_fp8_kvcache:
|
||||
assert metadata.scheduler_metadata.tile_scheduler_metadata is not None
|
||||
assert metadata.scheduler_metadata.num_splits is not None
|
||||
assert fp8_call is not None
|
||||
assert fp8_call[0] is seq_lens
|
||||
assert fp8_call[1:] == (8, 1)
|
||||
else:
|
||||
assert metadata.scheduler_metadata.tile_scheduler_metadata is None
|
||||
assert metadata.scheduler_metadata.num_splits is None
|
||||
assert fp8_call is None
|
||||
|
||||
|
||||
def run_attention_backend(
|
||||
backend: AttentionBackendEnum,
|
||||
kv_cache_spec: MLAAttentionSpec,
|
||||
|
||||
@@ -800,9 +800,9 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
else:
|
||||
mqa_q = (mqa_ql_nope, mqa_q_pe)
|
||||
if self.impl.dcp_world_size > 1:
|
||||
assert not fp8_attention, "DCP not support fp8 kvcache now."
|
||||
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
|
||||
mqa_q = torch.cat(mqa_q, dim=-1)
|
||||
if isinstance(mqa_q, tuple):
|
||||
# concatenate mqa_ql_nope and mqa_q_pe -> (B, N, L + P)
|
||||
mqa_q = torch.cat(mqa_q, dim=-1)
|
||||
# mqa_q do allgather in head dim.
|
||||
mqa_q = get_dcp_group().all_gather(mqa_q, dim=1)
|
||||
|
||||
@@ -1263,6 +1263,7 @@ class MLACommonPrefillMetadata:
|
||||
padded_local_chunk_seq_lens: list[list[int]] | None = None
|
||||
local_context_lens_allranks: list[list[int]] | None = None
|
||||
padded_local_cu_seq_lens: torch.Tensor | None = None
|
||||
padded_local_token_to_seq: torch.Tensor | None = None
|
||||
cu_seq_lens_lst: list[list[int]] | None = None
|
||||
chunk_size: int | None = None
|
||||
prefill_tokens_with_context: int | None = None
|
||||
@@ -1787,6 +1788,21 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
out=padded_local_cu_chunk_seq_lens_cpu[:, 1:],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
max_padded_local_tokens_over_chunk = (
|
||||
padded_local_cu_chunk_seq_lens_cpu[:, -1].max().item()
|
||||
)
|
||||
padded_local_token_to_seq_tensor_cpu = torch.zeros(
|
||||
[num_chunks, max_padded_local_tokens_over_chunk],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
for i in range(num_chunks):
|
||||
chunk_token_to_seq_tensor = torch.repeat_interleave(
|
||||
range_idx, padded_local_chunk_seq_lens[i]
|
||||
)
|
||||
chunk_len = chunk_token_to_seq_tensor.shape[0]
|
||||
padded_local_token_to_seq_tensor_cpu[i, :chunk_len] = (
|
||||
chunk_token_to_seq_tensor
|
||||
)
|
||||
|
||||
prefill_tokens_with_context = None
|
||||
if num_prefills_with_context_cpu > 0:
|
||||
@@ -1811,6 +1827,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
padded_local_cu_seq_lens=padded_local_cu_chunk_seq_lens_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
padded_local_token_to_seq=padded_local_token_to_seq_tensor_cpu.to(
|
||||
device, non_blocking=True
|
||||
),
|
||||
cu_seq_lens_lst=cu_seq_lens_cpu.tolist(),
|
||||
chunk_size=padded_local_max_context_chunk_across_ranks,
|
||||
prefill_tokens_with_context=prefill_tokens_with_context,
|
||||
@@ -2187,7 +2206,6 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
k_scale: torch.Tensor,
|
||||
dcp_world_size: int,
|
||||
):
|
||||
assert k_scale is None, "DCP not support scaled kvcache now."
|
||||
assert attn_metadata.prefill is not None
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
assert prefill_metadata.prefill_backend is not None
|
||||
@@ -2195,9 +2213,11 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
assert prefill_metadata.chunked_context.padded_local_chunk_seq_lens is not None
|
||||
assert prefill_metadata.chunked_context.local_context_lens_allranks is not None
|
||||
assert prefill_metadata.chunked_context.padded_local_cu_seq_lens is not None
|
||||
assert prefill_metadata.chunked_context.padded_local_token_to_seq is not None
|
||||
assert prefill_metadata.chunked_context.cu_seq_lens_lst is not None
|
||||
assert prefill_metadata.chunked_context.chunk_size is not None
|
||||
|
||||
use_fp8_prefill = prefill_metadata.q_data_type == current_platform.fp8_dtype()
|
||||
output = None
|
||||
merge_output = None
|
||||
iters = len(prefill_metadata.chunked_context.seq_tot)
|
||||
@@ -2205,16 +2225,37 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
|
||||
for i in range(iters):
|
||||
toks = prefill_metadata.chunked_context.seq_tot[i]
|
||||
ops.cp_gather_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
cu_seq_lens=prefill_metadata.chunked_context.padded_local_cu_seq_lens[
|
||||
i
|
||||
],
|
||||
batch_size=attn_metadata.num_prefills,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
if toks == 0:
|
||||
continue
|
||||
padded_local_cu_seq_lens = (
|
||||
prefill_metadata.chunked_context.padded_local_cu_seq_lens[i]
|
||||
)
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype) and (
|
||||
self.kv_cache_dtype != "fp8_ds_mla"
|
||||
):
|
||||
assert k_scale is not None
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
cu_seq_lens=padded_local_cu_seq_lens,
|
||||
token_to_seq=prefill_metadata.chunked_context.padded_local_token_to_seq[
|
||||
i
|
||||
],
|
||||
num_tokens=toks,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
scale=k_scale,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
)
|
||||
else:
|
||||
ops.cp_gather_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
block_table=prefill_metadata.block_table,
|
||||
cu_seq_lens=padded_local_cu_seq_lens,
|
||||
batch_size=attn_metadata.num_prefills,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
)
|
||||
# workspace
|
||||
# |------- N tokens --------|--------- N*dcp_size tokens ----------|
|
||||
# |<- use for local_gather ->|<--------- use for allgather -------->|
|
||||
@@ -2252,9 +2293,22 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
toks=toks,
|
||||
)
|
||||
|
||||
kv_b_proj_w_dtype = (
|
||||
self.kv_b_proj.weight.dtype
|
||||
if hasattr(self.kv_b_proj, "weight")
|
||||
else self.kv_b_proj.params_dtype
|
||||
)
|
||||
if (
|
||||
use_fp8_prefill or kv_b_proj_w_dtype != current_platform.fp8_dtype()
|
||||
) and kv_b_proj_w_dtype != torch.uint8:
|
||||
kv_c_normed = kv_c_normed.to(kv_b_proj_w_dtype)
|
||||
|
||||
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
|
||||
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
|
||||
)
|
||||
if use_fp8_prefill:
|
||||
kv_nope = kv_nope.to(prefill_metadata.q_data_type)
|
||||
k_pe = k_pe.to(prefill_metadata.q_data_type)
|
||||
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
|
||||
k = self._concat_k_nope_k_pe(k_nope, k_pe)
|
||||
|
||||
@@ -2346,7 +2400,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
attn_metadata,
|
||||
k_scale=None,
|
||||
k_scale=k_scale,
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -171,7 +171,10 @@ class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]):
|
||||
query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
# we use the max but all should be the same due to uniform length requirement
|
||||
max_query_len = query_lens_cpu.max().item()
|
||||
num_q_tokens_per_head_k = max_query_len * self.num_q_heads // 1
|
||||
num_q_heads = self.num_q_heads
|
||||
if self.dcp_world_size > 1:
|
||||
num_q_heads *= self.dcp_world_size
|
||||
num_q_tokens_per_head_k = max_query_len * num_q_heads // 1
|
||||
scheduler_metadata, _ = get_mla_metadata(
|
||||
seq_lens_device,
|
||||
num_q_tokens_per_head_k,
|
||||
|
||||
Reference in New Issue
Block a user