Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -551,28 +551,33 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
raise ValueError(
|
||||
f"VLLM_GDN_DECODE_KERNEL=cuda is not supported: {reason}"
|
||||
)
|
||||
logger.info_once(
|
||||
"Falling back to the Triton GDN decode path: %s", reason
|
||||
)
|
||||
if self.speculative_config is not None:
|
||||
logger.info_once(
|
||||
"Falling back to the Triton GDN spec decode path: %s",
|
||||
reason,
|
||||
)
|
||||
self.gdn_decode_kernel = "triton"
|
||||
elif current_platform.is_cpu():
|
||||
self.gdn_decode_kernel = "CPU"
|
||||
elif current_platform.is_xpu():
|
||||
self.gdn_decode_kernel = "XPU"
|
||||
|
||||
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
|
||||
self.enable_fused_gdn_spec_decode = (
|
||||
self.gdn_decode_kernel == "cuda" and self.speculative_config is not None
|
||||
)
|
||||
# RecoverSSM verify replaces the per-draft snapshot path (Triton decode
|
||||
# only)
|
||||
self.use_gdn_recoverssm = self._uses_gdn_recoverssm()
|
||||
if self.use_gdn_recoverssm:
|
||||
self.gdn_decode_kernel = "triton"
|
||||
self.enable_fused_gdn_decode = False
|
||||
self.enable_fused_gdn_spec_decode = False
|
||||
self.enable_packed_recurrent_decode = False
|
||||
logger.info_once(
|
||||
"GDN RecoverSSM speculative verify active (spec_query_len %d)",
|
||||
self.num_spec + 1,
|
||||
)
|
||||
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
|
||||
if self.speculative_config is not None:
|
||||
logger.info_once("GDN spec decode kernel: %s", self.gdn_decode_kernel)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
@@ -969,12 +974,12 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
ba, _ = self.in_proj_ba(hidden_states)
|
||||
|
||||
use_fused_gdn_decode = (
|
||||
self.enable_fused_gdn_decode
|
||||
use_fused_gdn_spec_decode = (
|
||||
self.enable_fused_gdn_spec_decode
|
||||
and hidden_states.dtype == torch.bfloat16
|
||||
and self.norm.weight.dtype in (torch.bfloat16, torch.float32)
|
||||
)
|
||||
if use_fused_gdn_decode:
|
||||
if use_fused_gdn_spec_decode:
|
||||
core_attn_out = torch.zeros(
|
||||
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
|
||||
Reference in New Issue
Block a user