[CPU][MLA] Fix prefill backend selection so MLA runs end-to-end on CPU (#51471)
Signed-off-by: baoloongmao <baoloongmao@tencent.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
@@ -2785,6 +2785,18 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
batch_size=chunk.num_requests,
|
||||
seq_starts=chunk.starts,
|
||||
)
|
||||
elif current_platform.is_cpu():
|
||||
assert not is_quantized_kv_cache(self.kv_cache_dtype), (
|
||||
"CPU MLA context gather fallback only supports "
|
||||
f"non-quantized KV cache, got {self.kv_cache_dtype}"
|
||||
)
|
||||
ops.gather_mla_context_cache_cpu(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=block_table,
|
||||
starts=chunk.starts,
|
||||
cu_seq_lens=chunk.cu_seq_lens,
|
||||
)
|
||||
elif not use_fp8_prefill:
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
|
||||
Reference in New Issue
Block a user