[Misc] Fix Current vLLM config is not set. warnings, assert to avoid issues in the future (#31747)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
co-authored by
Luka Govedič
parent
5d3b6097ad
commit
6cdf015c3c
@@ -13,10 +13,28 @@ from vllm.model_executor.layers.quantization.utils.layer_utils import replace_pa
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
_CPU_MOE_LAYER_CACHE = {}
|
||||
_CPU_MOE_ACT = {
|
||||
"silu": SiluAndMul(),
|
||||
"swigluoai": SwigluOAIAndMul(),
|
||||
}
|
||||
|
||||
|
||||
class _LazyActivationDict(dict):
|
||||
"""Lazily instantiate activation functions on first access.
|
||||
|
||||
Avoids triggering CustomOp.__init__() at module import time,
|
||||
which would call get_current_vllm_config() before config is set.
|
||||
"""
|
||||
|
||||
_factories: dict[str, type[SiluAndMul] | type[SwigluOAIAndMul]] = {
|
||||
"silu": SiluAndMul,
|
||||
"swigluoai": SwigluOAIAndMul,
|
||||
}
|
||||
|
||||
def __missing__(self, key: str) -> SiluAndMul | SwigluOAIAndMul:
|
||||
if key not in self._factories:
|
||||
raise KeyError(f"{key} is not a supported activation")
|
||||
self[key] = self._factories[key]()
|
||||
return self[key]
|
||||
|
||||
|
||||
_CPU_MOE_ACT = _LazyActivationDict()
|
||||
|
||||
|
||||
def grouped_topk(
|
||||
@@ -212,7 +230,7 @@ class CPUFusedMOE:
|
||||
apply_router_weight_on_input: bool = False,
|
||||
activation: str = "silu",
|
||||
) -> torch.Tensor:
|
||||
assert activation in _CPU_MOE_ACT, f"{activation} is not supported."
|
||||
assert activation in _CPU_MOE_ACT._factories, f"{activation} is not supported."
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
|
||||
@@ -540,6 +540,20 @@ class FusedMoE(CustomOp):
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
self.activation = activation
|
||||
|
||||
self._grouped_topk_impl: GroupedTopk | None = None
|
||||
if self.use_grouped_topk:
|
||||
assert self.num_expert_group is not None
|
||||
assert self.topk_group is not None
|
||||
self._grouped_topk_impl = GroupedTopk(
|
||||
topk=self.top_k,
|
||||
renormalize=self.renormalize,
|
||||
num_expert_group=self.num_expert_group,
|
||||
topk_group=self.topk_group,
|
||||
scoring_func=self.scoring_func,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
)
|
||||
|
||||
if self.scoring_func != "softmax" and not self.use_grouped_topk:
|
||||
raise ValueError(
|
||||
"Only softmax scoring function is supported for non-grouped topk."
|
||||
@@ -1588,19 +1602,8 @@ class FusedMoE(CustomOp):
|
||||
|
||||
# DeepSeekv2 uses grouped_top_k
|
||||
elif self.use_grouped_topk and valid_grouping():
|
||||
assert self.topk_group is not None
|
||||
assert self.num_expert_group is not None
|
||||
grouped_topk_impl = GroupedTopk(
|
||||
topk=self.top_k,
|
||||
renormalize=self.renormalize,
|
||||
num_expert_group=self.num_expert_group,
|
||||
topk_group=self.topk_group,
|
||||
scoring_func=self.scoring_func,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = grouped_topk_impl(
|
||||
assert self._grouped_topk_impl is not None
|
||||
topk_weights, topk_ids = self._grouped_topk_impl(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
e_score_correction_bias=self.e_score_correction_bias,
|
||||
|
||||
@@ -339,15 +339,11 @@ def apply_rotary_pos_emb_flashatt(
|
||||
k: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
apply_rotary_emb: ApplyRotaryEmb,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
cos = cos.chunk(2, dim=-1)[0].contiguous()
|
||||
sin = sin.chunk(2, dim=-1)[0].contiguous()
|
||||
|
||||
apply_rotary_emb = ApplyRotaryEmb(
|
||||
enforce_enable=True,
|
||||
enable_fp32_compute=True,
|
||||
)
|
||||
|
||||
q_embed = apply_rotary_emb(q, cos, sin)
|
||||
k_embed = apply_rotary_emb(k, cos, sin)
|
||||
|
||||
@@ -410,6 +406,11 @@ class KeyeSiglipAttention(nn.Module):
|
||||
multimodal_config=multimodal_config,
|
||||
)
|
||||
|
||||
self.apply_rotary_emb = ApplyRotaryEmb(
|
||||
enforce_enable=True,
|
||||
enable_fp32_compute=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -448,7 +449,7 @@ class KeyeSiglipAttention(nn.Module):
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
)
|
||||
q, k = apply_rotary_pos_emb_flashatt(q, k, cos, sin)
|
||||
q, k = apply_rotary_pos_emb_flashatt(q, k, cos, sin, self.apply_rotary_emb)
|
||||
v = v.view(
|
||||
*v.shape[:-1],
|
||||
self.num_kv_heads,
|
||||
|
||||
@@ -152,16 +152,12 @@ def apply_rotary_pos_emb(
|
||||
k: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
is_flash_attn_backend: bool = False,
|
||||
is_flash_attn_backend: bool,
|
||||
apply_rotary_emb: ApplyRotaryEmb,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
cos = cos.chunk(2, dim=-1)[0].contiguous()
|
||||
sin = sin.chunk(2, dim=-1)[0].contiguous()
|
||||
|
||||
apply_rotary_emb = ApplyRotaryEmb(
|
||||
enforce_enable=True,
|
||||
enable_fp32_compute=True,
|
||||
)
|
||||
|
||||
if is_flash_attn_backend and current_platform.is_cuda():
|
||||
apply_rotary_emb_func = apply_rotary_emb.forward_cuda
|
||||
elif is_flash_attn_backend and current_platform.is_rocm():
|
||||
@@ -235,6 +231,11 @@ class Siglip2Attention(nn.Module):
|
||||
multimodal_config=multimodal_config,
|
||||
)
|
||||
|
||||
self.apply_rotary_emb = ApplyRotaryEmb(
|
||||
enforce_enable=True,
|
||||
enable_fp32_compute=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -260,6 +261,7 @@ class Siglip2Attention(nn.Module):
|
||||
cos,
|
||||
sin,
|
||||
self.attn.is_flash_attn_backend,
|
||||
self.apply_rotary_emb,
|
||||
)
|
||||
queries = queries.squeeze(0)
|
||||
keys = keys.squeeze(0)
|
||||
|
||||
Reference in New Issue
Block a user