[XPU] Initial support for GDN attention on Qwen3-next/Qwen3.5 (#33657)
Signed-off-by: Yan Ma <yan.ma@intel.com> Signed-off-by: Chendi Xue <chendi.xue@intel.com> Co-authored-by: Chendi Xue <chendi.xue@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Chendi Xue
Kunshang Ji
parent
05e68e1f81
commit
ee3cf45739
@@ -218,6 +218,57 @@ class XPUPlatform(Platform):
|
||||
# ref. https://openucx.readthedocs.io/en/master/faq.html
|
||||
os.environ["UCX_MEMTYPE_CACHE"] = "n"
|
||||
|
||||
@classmethod
|
||||
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
|
||||
super().update_block_size_for_backend(vllm_config)
|
||||
from vllm.config.vllm import get_layers_from_vllm_config
|
||||
from vllm.model_executor.layers.attention_layer_base import (
|
||||
AttentionLayerBase,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
# special fix for GDN since kernel only supports block size dividable by 64
|
||||
attn_layers = get_layers_from_vllm_config(
|
||||
vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
)
|
||||
|
||||
kernel_block_size = None
|
||||
for layer in attn_layers.values():
|
||||
b = layer.get_attn_backend()
|
||||
if b.get_name() == "GDN_ATTN":
|
||||
kernel_block_size = 64
|
||||
break
|
||||
|
||||
if kernel_block_size is None:
|
||||
return
|
||||
new_block_size = (
|
||||
cdiv(cache_config.block_size, kernel_block_size) * kernel_block_size
|
||||
)
|
||||
if new_block_size == cache_config.block_size:
|
||||
return
|
||||
|
||||
if cache_config.mamba_cache_mode == "align":
|
||||
cache_config.mamba_block_size = new_block_size
|
||||
original_mamba_page_size_padded = cache_config.mamba_page_size_padded
|
||||
if cache_config.mamba_page_size_padded is not None:
|
||||
attn_page_size_1_token = (
|
||||
cache_config.mamba_page_size_padded // cache_config.block_size
|
||||
)
|
||||
cache_config.mamba_page_size_padded = (
|
||||
new_block_size * attn_page_size_1_token
|
||||
)
|
||||
cache_config.block_size = new_block_size
|
||||
logger.info(
|
||||
"[XPU]Setting attention block size to %d tokens to ensure multiple of %d, "
|
||||
"set mamba_page_size_padded to %d bytes accordingly, before was %d bytes.",
|
||||
new_block_size,
|
||||
kernel_block_size,
|
||||
cache_config.mamba_page_size_padded,
|
||||
original_mamba_page_size_padded,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def support_hybrid_kv_cache(cls) -> bool:
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user