[Bugfix][RL] Load attention sinks through weight_loader for layerwise reload (#57797)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -33,7 +33,11 @@ from vllm.config.load import (
|
||||
DEFAULT_SAFETENSORS_PREFETCH_NUM_THREADS,
|
||||
LoadConfig,
|
||||
)
|
||||
from vllm.distributed import get_tensor_model_parallel_rank, get_world_group
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_world_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
@@ -1493,6 +1497,33 @@ def row_parallel_weight_loader(
|
||||
LoaderFunction = Callable[[torch.Tensor, torch.Tensor], None]
|
||||
|
||||
|
||||
def load_attention_sink(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
"""Load this rank's heads of a global sink; padded heads are set to -inf."""
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
heads = loaded_weight.numel()
|
||||
if heads % tp_size:
|
||||
raise ValueError(
|
||||
f"Attention sink heads ({heads}) must be divisible by tp_size ({tp_size})"
|
||||
)
|
||||
local_heads = heads // tp_size
|
||||
start = get_tensor_model_parallel_rank() * local_heads
|
||||
default_weight_loader(
|
||||
param[:local_heads], loaded_weight.narrow(0, start, local_heads)
|
||||
)
|
||||
param.data[local_heads:].fill_(-float("inf"))
|
||||
|
||||
|
||||
def make_attention_sink(
|
||||
num_heads: int, dtype: torch.dtype | None = None
|
||||
) -> torch.nn.Parameter:
|
||||
"""Create a sink with -inf padding and register its TP weight loader."""
|
||||
sink = torch.nn.Parameter(
|
||||
torch.full((num_heads,), -float("inf"), dtype=dtype), requires_grad=False
|
||||
)
|
||||
sink.weight_loader = load_attention_sink
|
||||
return sink
|
||||
|
||||
|
||||
def sharded_weight_loader(shard_axis: int) -> LoaderFunction:
|
||||
"""Create a weight loader that shards the weights along the given axis"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user