[HPC Attention Backend] hpc attention backend support bf16 kv cache with fp8 weight (#50980)

Signed-off-by: chengvjiang <chengvjiang@tencent.com>
Co-authored-by: chengvjiang <chengvjiang@tencent.com>
Co-authored-by: VAthree <56064364+VAthree@users.noreply.github.com>
This commit is contained in:
Cheng Jiang
2026-08-06 15:40:14 +08:00
committed by GitHub
co-authored by chengvjiang VAthree
parent 7e724dca89
commit 470297c143
3 changed files with 75 additions and 36 deletions
@@ -62,11 +62,7 @@ class HPCExperts(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_current_device() -> bool:
p = current_platform
return (
p.is_cuda()
and (p.is_device_capability(90) or p.is_device_capability_family(100))
and has_hpc()
)
return p.is_cuda() and p.is_device_capability(90) and has_hpc()
@staticmethod
def _supports_no_act_and_mul() -> bool:
+10 -2
View File
@@ -205,10 +205,10 @@ class HpcRopeNorm(CustomOp, HpcModule):
):
return False
if kv_cache_dtype not in ("fp8_e4m3", "auto"):
if kv_cache_dtype not in ("fp8_e4m3", "auto", "bfloat16"):
logger.warning_once(
f"hpc rope_norm not support kv_cache_dtype:{kv_cache_dtype}, "
"only support fp8_e4m3, bfloat16"
"only support fp8_e4m3, auto, bfloat16"
)
return False
@@ -414,3 +414,11 @@ class HpcRopeNorm(CustomOp, HpcModule):
out_q=out_q_decode,
qk_norm_policy=self.qk_norm_policy,
)
# Signal HpcAttentionImpl that KV cache has already been written by
# rope_norm_store_kv[_fp8] above, so it should skip its own
# reshape_and_cache_flash. Set after the fused kernels ran (either
# prefill or decode branch); otherwise the standard KV-write path in
# the attention impl would kick in (which is what non-HpcRopeNorm
# models rely on).
attn_metadata.hpc_kv_written = True
+64 -29
View File
@@ -80,7 +80,7 @@ class HpcAttnMetadata(AttentionMetadata):
# HpcAttentionImpl.forward(). Defaults are safe for the standard
# (non-RopeNorm) path and for profiling runs (attn_metadata=None).
hpc_kv_written: bool = False
"""True when HpcRopeNorm already wrote KV cache."""
"""True when HpcRopeNorm has already written K/V into the paged cache."""
hpc_prefill_q_scale: torch.Tensor | None = None
"""FP8 per-token-per-head Q scale for prefill (from RopeNorm)."""
hpc_decode_q_scale: torch.Tensor | None = None
@@ -146,12 +146,24 @@ class HpcAttnMetadataBuilder(AttentionMetadataBuilder[HpcAttnMetadata]):
supports_spec_as_decode=True,
)
self.task_map = hpc.get_attention_decode_task_workspace(
vllm_config.scheduler_config.max_num_seqs,
vllm_config.model_config.max_model_len or 4096,
self.num_kv_heads,
min_process_len=self.hpc_dynamic_sched_attn_min_split_len,
)
# Dynamic-scheduled decode:
# - fp8 KV cache path always uses task_map (FP8 decode kernel
# requires it).
# - bf16 KV cache path only uses task_map when the installed hpc
# version supports it (hpc >= 6e2eced / PR #73). Older builds
# take the static split-K path with no task_map.
kv_cache_dtype = kv_cache_spec.dtype
self.use_fp8 = kv_cache_dtype == torch.float8_e4m3fn
self._dynamic_sched = True
if self._dynamic_sched:
self.task_map = hpc.get_attention_decode_task_workspace(
vllm_config.scheduler_config.max_num_seqs,
vllm_config.model_config.max_model_len or 4096,
self.num_kv_heads,
min_process_len=self.hpc_dynamic_sched_attn_min_split_len,
)
else:
self.task_map = None
@override # type: ignore[misc]
@classmethod
@@ -214,16 +226,20 @@ class HpcAttnMetadataBuilder(AttentionMetadataBuilder[HpcAttnMetadata]):
seq_lens_decode = seq_lens[:num_decodes]
# block_table is per-request, indexed by num_decodes (not tokens)
qo_indptr_decode = common_attn_metadata.query_start_loc[: num_decodes + 1]
import hpc
hpc.assign_attention_decode_task(
seq_lens_decode,
self.task_map,
self.num_kv_heads,
decode_query_len,
new_kv_included=True,
min_process_len=self.hpc_dynamic_sched_attn_min_split_len,
)
# Skip the task-map scheduler pass when dynamic scheduling is off
# (bf16 KV cache on an older hpc that lacks bf16 decode task_map).
if self._dynamic_sched:
import hpc
hpc.assign_attention_decode_task(
seq_lens_decode,
self.task_map,
self.num_kv_heads,
decode_query_len,
new_kv_included=True,
min_process_len=self.hpc_dynamic_sched_attn_min_split_len,
)
return HpcAttnMetadata(
num_actual_tokens=num_actual_tokens,
@@ -236,7 +252,7 @@ class HpcAttnMetadataBuilder(AttentionMetadataBuilder[HpcAttnMetadata]):
seq_lens=seq_lens,
block_table_tensor=block_table_tensor,
qo_indptr=qo_indptr,
hpc_kv_written=True,
hpc_kv_written=False,
hpc_prefill_q_scale=None,
hpc_decode_q_scale=None,
hpc_split_k_flag=None,
@@ -306,7 +322,7 @@ class HpcAttentionBackend(AttentionBackend):
@classmethod
def supports_compute_capability(cls, capability: DeviceCapability) -> bool:
return capability >= DeviceCapability(9, 0)
return capability == DeviceCapability(9, 0)
@classmethod
def supports_kv_cache_dtype(cls, kv_cache_dtype: "CacheDType | None") -> bool:
@@ -366,10 +382,10 @@ class HpcAttentionImpl(AttentionImpl[HpcAttnMetadata]):
f"(num_heads={num_heads}, num_kv_heads={num_kv_heads})"
)
if kv_cache_dtype not in ("auto", "fp8_e4m3"):
if kv_cache_dtype not in ("auto", "fp8_e4m3", "bfloat16"):
raise ValueError(
f"HPC attention only supports kv_cache_dtype 'auto' or "
f"'fp8_e4m3', got '{kv_cache_dtype}'"
f"'fp8_e4m3' or 'bfloat16', got '{kv_cache_dtype}'"
)
self.num_heads = num_heads
@@ -436,8 +452,17 @@ class HpcAttentionImpl(AttentionImpl[HpcAttnMetadata]):
num_decode_reqs = attn_metadata.num_decodes
num_decode_tokens = attn_metadata.num_decode_tokens
# Write KV cache if not already done by HpcRopeNorm.
if self.kv_sharing_target_layer_name is None and not hpc_kv_written:
# Standard-path KV write: when no upstream fused op wrote KV cache
# (i.e. the model didn't wire HpcRopeNorm), fall back to the generic
# reshape_and_cache_flash. This is what lets non-HunYuan-V3 models use
# the HPC attention backend with bf16 KV cache without any model-side
# changes. FP8 KV cache always needs HpcRopeNorm (see check below), so
# we never take this path in fp8 mode.
if (
not self.use_fp8
and self.kv_sharing_target_layer_name is None
and not hpc_kv_written
):
torch.ops._C_cache_ops.reshape_and_cache_flash(
key,
value,
@@ -449,18 +474,18 @@ class HpcAttentionImpl(AttentionImpl[HpcAttnMetadata]):
layer._v_scale,
)
if self.use_fp8:
torch_dtype = _get_fp8_dtype_for_kv_cache(self.kv_cache_dtype)
kv_cache = kv_cache.view(torch_dtype)
if self.use_fp8:
if not hpc_kv_written:
raise RuntimeError(
"HpcAttentionImpl: FP8 mode requires HpcRopeNorm. "
"Ensure hpc_rope_norm is enabled or set "
"kv_cache_dtype='auto' for bf16 mode."
"HpcAttentionImpl: FP8 KV cache mode requires HpcRopeNorm "
"(the fused rope_norm_store_kv_fp8 op produces the "
"per-token-per-head Q scales the FP8 attention kernel "
"needs). Ensure the model wires hpc_rope_norm, or switch "
"to --kv-cache-dtype auto / bfloat16."
f" (layer={getattr(layer, 'layer_name', '?')})"
)
torch_dtype = _get_fp8_dtype_for_kv_cache(self.kv_cache_dtype)
kv_cache = kv_cache.view(torch_dtype)
k_scale = layer._k_scale.reshape(1)
v_scale = layer._v_scale.reshape(1)
@@ -539,6 +564,15 @@ class HpcAttentionImpl(AttentionImpl[HpcAttnMetadata]):
output=output_decode,
)
else:
# Older hpc's bf16 decode has no task_map kwarg; the map is
# only ever built when the installed hpc supports it (see
# HpcAttnMetadataBuilder._dynamic_sched). Pass task_map
# conditionally to stay compatible with both versions
task_map_kwargs = (
{"task_map": attn_metadata.task_map}
if attn_metadata.task_map is not None
else {}
)
hpc.attention_decode_bf16(
q_decode,
kv_cache[:, 0],
@@ -549,6 +583,7 @@ class HpcAttentionImpl(AttentionImpl[HpcAttnMetadata]):
output=output_decode,
new_kv_included=True,
splitk=self.splitk,
**task_map_kwargs,
)
return output_padded