[ROCM][Perf] Add ROCM_SEGMENTED_ATTN attention-backend works across RDNA3/3.5/4 (#59132)
Signed-off-by: root <jeffaw99@hotmail.com> Signed-off-by: big-yellow-duck <jeffaw99@hotmail.com> Co-authored-by: Tan Pin Siang <1716735+tanpinsiang@users.noreply.github.com> Co-authored-by: vllmellm <190700713+vllmellm@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Tan Pin Siang
vllmellm
Codex
parent
15c1cc12ac
commit
edfccc9469
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.config import set_default_rope_theta
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
@@ -269,6 +270,24 @@ class DFlashQwen3Attention(nn.Module):
|
||||
sinks=self.attention_sink_bias,
|
||||
)
|
||||
self.causal = causal
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx1x
|
||||
from vllm.v1.attention.backends.rocm_segmented_attn import (
|
||||
RocmSegmentedAttentionImpl,
|
||||
)
|
||||
|
||||
if on_gfx1x() and isinstance(self.attn.impl, RocmSegmentedAttentionImpl):
|
||||
# Startup segmented tuning needs the draft's actual attention mode.
|
||||
# Causality is normally supplied later through attention metadata.
|
||||
self.attn.segmented_causal = causal
|
||||
speculative_config = get_current_vllm_config().speculative_config
|
||||
if (
|
||||
speculative_config is not None
|
||||
and speculative_config.num_speculative_tokens is not None
|
||||
):
|
||||
self.attn.segmented_query_limit = (
|
||||
speculative_config.num_speculative_tokens + 1
|
||||
)
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
|
||||
|
||||
@@ -220,6 +220,13 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
if process_local_only:
|
||||
return
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.model_executor.warmup.rocm_segmented_attn_autotune_warmup import (
|
||||
rocm_segmented_attn_autotune_warmup,
|
||||
)
|
||||
|
||||
rocm_segmented_attn_autotune_warmup(worker)
|
||||
|
||||
flashinfer_sparse_mla_decode_autotune_warmup(worker)
|
||||
deepseek_v4_sparse_mla_attention_warmup(worker)
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user