[CPU] Fix FP8 attention scratchpad sizing (#50194)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Codex
Li, Jiang <jiang1.li@intel.com>
parent
100d655a23
commit
65a1a16594
@@ -30,7 +30,8 @@ torch::Tensor get_scheduler_metadata(
|
|||||||
const torch::Tensor& query_start_loc, const bool causal,
|
const torch::Tensor& query_start_loc, const bool causal,
|
||||||
const int64_t window_size, const std::string& isa_hint,
|
const int64_t window_size, const std::string& isa_hint,
|
||||||
const bool enable_kv_split,
|
const bool enable_kv_split,
|
||||||
const std::optional<torch::Tensor>& dynamic_causal) {
|
const std::optional<torch::Tensor>& dynamic_causal,
|
||||||
|
const std::string& kv_cache_dtype) {
|
||||||
cpu_attention::ISA isa;
|
cpu_attention::ISA isa;
|
||||||
if (isa_hint == "amx") {
|
if (isa_hint == "amx") {
|
||||||
isa = cpu_attention::ISA::AMX;
|
isa = cpu_attention::ISA::AMX;
|
||||||
@@ -65,9 +66,11 @@ torch::Tensor get_scheduler_metadata(
|
|||||||
input.dynamic_causal =
|
input.dynamic_causal =
|
||||||
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
|
dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
|
||||||
|
|
||||||
|
const int64_t kv_cache_idx =
|
||||||
|
static_cast<int64_t>(parse_fp8_kv_dtype(kv_cache_dtype));
|
||||||
VLLM_DISPATCH_FLOATING_TYPES(dtype, "get_scheduler_metadata", [&]() {
|
VLLM_DISPATCH_FLOATING_TYPES(dtype, "get_scheduler_metadata", [&]() {
|
||||||
CPU_ATTN_DISPATCH(head_dim, isa, 0, [&]() {
|
CPU_ATTN_DISPATCH(head_dim, isa, kv_cache_idx, [&]() {
|
||||||
input.elem_size = sizeof(scalar_t);
|
input.elem_size = sizeof(attn_impl::kv_cache_t);
|
||||||
input.q_buffer_elem_size = sizeof(attn_impl::q_buffer_t);
|
input.q_buffer_elem_size = sizeof(attn_impl::q_buffer_t);
|
||||||
input.logits_buffer_elem_size = sizeof(attn_impl::logits_buffer_t);
|
input.logits_buffer_elem_size = sizeof(attn_impl::logits_buffer_t);
|
||||||
input.output_buffer_elem_size =
|
input.output_buffer_elem_size =
|
||||||
|
|||||||
@@ -163,7 +163,8 @@ torch::Tensor get_scheduler_metadata(
|
|||||||
const torch::Tensor& query_start_loc, const bool casual,
|
const torch::Tensor& query_start_loc, const bool casual,
|
||||||
const int64_t window_size, const std::string& isa_hint,
|
const int64_t window_size, const std::string& isa_hint,
|
||||||
const bool enable_kv_split,
|
const bool enable_kv_split,
|
||||||
const std::optional<torch::Tensor>& dynamic_causal);
|
const std::optional<torch::Tensor>& dynamic_causal,
|
||||||
|
const std::string& kv_cache_dtype);
|
||||||
|
|
||||||
void cpu_attn_reshape_and_cache(const torch::Tensor& key,
|
void cpu_attn_reshape_and_cache(const torch::Tensor& key,
|
||||||
const torch::Tensor& value,
|
const torch::Tensor& value,
|
||||||
@@ -577,7 +578,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
|
|||||||
"get_scheduler_metadata(int num_req, int num_heads_q, int num_heads_kv, "
|
"get_scheduler_metadata(int num_req, int num_heads_q, int num_heads_kv, "
|
||||||
"int head_dim, Tensor seq_lens, ScalarType dtype, Tensor "
|
"int head_dim, Tensor seq_lens, ScalarType dtype, Tensor "
|
||||||
"query_start_loc, bool casual, int window_size, str isa_hint, bool "
|
"query_start_loc, bool casual, int window_size, str isa_hint, bool "
|
||||||
"enable_kv_split, Tensor? dynamic_causal) -> Tensor",
|
"enable_kv_split, Tensor? dynamic_causal, "
|
||||||
|
"str kv_cache_dtype=\"auto\") -> Tensor",
|
||||||
&get_scheduler_metadata);
|
&get_scheduler_metadata);
|
||||||
ops.def(
|
ops.def(
|
||||||
"cpu_attn_reshape_and_cache(Tensor key, Tensor value, Tensor(a2!) "
|
"cpu_attn_reshape_and_cache(Tensor key, Tensor value, Tensor(a2!) "
|
||||||
|
|||||||
@@ -534,6 +534,7 @@ def varlen_with_paged_kv(
|
|||||||
isa=isa,
|
isa=isa,
|
||||||
enable_kv_split=False,
|
enable_kv_split=False,
|
||||||
dynamic_causal=dynamic_causal_tensor,
|
dynamic_causal=dynamic_causal_tensor,
|
||||||
|
kv_cache_dtype=kv_cache_dtype,
|
||||||
)
|
)
|
||||||
|
|
||||||
out_without_split = torch.empty_like(query)
|
out_without_split = torch.empty_like(query)
|
||||||
@@ -569,6 +570,7 @@ def varlen_with_paged_kv(
|
|||||||
isa=isa,
|
isa=isa,
|
||||||
enable_kv_split=True,
|
enable_kv_split=True,
|
||||||
dynamic_causal=dynamic_causal_tensor,
|
dynamic_causal=dynamic_causal_tensor,
|
||||||
|
kv_cache_dtype=kv_cache_dtype,
|
||||||
)
|
)
|
||||||
|
|
||||||
out_with_split = torch.empty_like(query)
|
out_with_split = torch.empty_like(query)
|
||||||
@@ -803,6 +805,24 @@ def test_varlen_with_paged_kv_normal_amx(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not torch.cpu._is_amx_tile_supported(), reason="no AMX support.")
|
||||||
|
def test_varlen_with_paged_kv_fp8_large_prefill_amx() -> None:
|
||||||
|
varlen_with_paged_kv(
|
||||||
|
seq_lens=[(1024, 1024)] * 4,
|
||||||
|
num_heads=(16, 2),
|
||||||
|
head_size=256,
|
||||||
|
sliding_window=None,
|
||||||
|
dtype=torch.bfloat16,
|
||||||
|
block_size=2176,
|
||||||
|
soft_cap=None,
|
||||||
|
num_blocks=4,
|
||||||
|
use_alibi=False,
|
||||||
|
use_sink=False,
|
||||||
|
isa="amx",
|
||||||
|
kv_cache_dtype="fp8_e4m3",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("seq_lens", SEQ_LENS)
|
@pytest.mark.parametrize("seq_lens", SEQ_LENS)
|
||||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||||
@pytest.mark.parametrize("head_size", HEAD_SIZES_VEC16)
|
@pytest.mark.parametrize("head_size", HEAD_SIZES_VEC16)
|
||||||
|
|||||||
@@ -3814,6 +3814,7 @@ def cpu_attn_get_scheduler_metadata(
|
|||||||
isa: str,
|
isa: str,
|
||||||
enable_kv_split: bool,
|
enable_kv_split: bool,
|
||||||
dynamic_causal: torch.Tensor | None = None,
|
dynamic_causal: torch.Tensor | None = None,
|
||||||
|
kv_cache_dtype: str = "auto",
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
scheduler_metadata = torch.ops._C.get_scheduler_metadata(
|
scheduler_metadata = torch.ops._C.get_scheduler_metadata(
|
||||||
num_reqs,
|
num_reqs,
|
||||||
@@ -3828,6 +3829,7 @@ def cpu_attn_get_scheduler_metadata(
|
|||||||
isa,
|
isa,
|
||||||
enable_kv_split,
|
enable_kv_split,
|
||||||
dynamic_causal,
|
dynamic_causal,
|
||||||
|
kv_cache_dtype,
|
||||||
)
|
)
|
||||||
return scheduler_metadata
|
return scheduler_metadata
|
||||||
|
|
||||||
|
|||||||
@@ -155,12 +155,12 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
|||||||
if self.window_size is None:
|
if self.window_size is None:
|
||||||
self.window_size = -1
|
self.window_size = -1
|
||||||
self.block_size = vllm_config.cache_config.block_size
|
self.block_size = vllm_config.cache_config.block_size
|
||||||
kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
|
self.kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
||||||
self.isa = _get_attn_isa(
|
self.isa = _get_attn_isa(
|
||||||
self.dtype,
|
self.dtype,
|
||||||
self.block_size,
|
self.block_size,
|
||||||
self.head_dim,
|
self.head_dim,
|
||||||
kv_cache_dtype_str,
|
self.kv_cache_dtype,
|
||||||
)
|
)
|
||||||
self.is_cross_attention = isinstance(kv_cache_spec, CrossAttentionSpec)
|
self.is_cross_attention = isinstance(kv_cache_spec, CrossAttentionSpec)
|
||||||
self.is_encoder_only_attention = isinstance(
|
self.is_encoder_only_attention = isinstance(
|
||||||
@@ -234,6 +234,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata]
|
|||||||
isa=self.isa,
|
isa=self.isa,
|
||||||
enable_kv_split=envs.VLLM_CPU_ATTN_SPLIT_KV,
|
enable_kv_split=envs.VLLM_CPU_ATTN_SPLIT_KV,
|
||||||
dynamic_causal=dynamic_casual,
|
dynamic_causal=dynamic_casual,
|
||||||
|
kv_cache_dtype=self.kv_cache_dtype,
|
||||||
)
|
)
|
||||||
|
|
||||||
attn_metadata = CPUAttentionMetadata(
|
attn_metadata = CPUAttentionMetadata(
|
||||||
|
|||||||
Reference in New Issue
Block a user