[CI/Build][Bugfix] Fix SD LoRA (#45941)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import torch.nn as nn
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.distributed.parallel_state import get_pp_group
|
from vllm.distributed.parallel_state import get_pp_group
|
||||||
|
from vllm.lora.layers.base import BaseLayerWithLoRA
|
||||||
from vllm.model_executor.model_loader import get_model
|
from vllm.model_executor.model_loader import get_model
|
||||||
|
|
||||||
|
|
||||||
@@ -48,6 +49,13 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
|||||||
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
|
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
|
||||||
target_inner, "embedding", None
|
target_inner, "embedding", None
|
||||||
)
|
)
|
||||||
|
# If the target's embedding is LoRA-wrapped, share the underlying base
|
||||||
|
# layer. The draft is not part of the LoRA adapter; sharing the wrapper
|
||||||
|
# would make the draft run the LoRA embedding kernel with the target's
|
||||||
|
# punica metadata (sized for the target's token count), causing an
|
||||||
|
# out-of-bounds GPU access during multi-step draft decode.
|
||||||
|
if isinstance(target_embed, BaseLayerWithLoRA):
|
||||||
|
target_embed = target_embed.base_layer
|
||||||
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
||||||
if target_embed is not None and _should_share(
|
if target_embed is not None and _should_share(
|
||||||
eagle_model, "has_own_embed_tokens", draft_embed, target_embed
|
eagle_model, "has_own_embed_tokens", draft_embed, target_embed
|
||||||
|
|||||||
Reference in New Issue
Block a user