[Bugfix][RL] Load attention sinks through weight_loader for layerwise reload (#57797)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -9,15 +9,107 @@ import pytest
|
||||
import torch
|
||||
from huggingface_hub.utils import LocalEntryNotFoundError
|
||||
|
||||
import vllm.model_executor.model_loader.weight_utils as weight_utils
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf,
|
||||
drop_checkpoint_cache,
|
||||
initialize_single_dummy_weight,
|
||||
load_attention_sink,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_heads,runtime_heads,tp_size",
|
||||
[(8, 4, 2), (8, 8, 1), (8, 6, 2), (16, 12, 4)],
|
||||
)
|
||||
def test_attention_sink_loader_shapes(
|
||||
monkeypatch, checkpoint_heads, runtime_heads, tp_size
|
||||
):
|
||||
"""Each rank loads its checkpoint heads and disables only padded heads."""
|
||||
monkeypatch.setattr(
|
||||
weight_utils, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
checkpoint = torch.arange(checkpoint_heads, dtype=torch.float32)
|
||||
for tp_rank in range(tp_size):
|
||||
monkeypatch.setattr(
|
||||
weight_utils, "get_tensor_model_parallel_rank", lambda rank=tp_rank: rank
|
||||
)
|
||||
param = torch.nn.Parameter(torch.zeros(runtime_heads), requires_grad=False)
|
||||
load_attention_sink(param, checkpoint)
|
||||
local_heads = checkpoint.chunk(tp_size)[tp_rank]
|
||||
expected = torch.cat(
|
||||
(
|
||||
local_heads,
|
||||
torch.full((runtime_heads - local_heads.numel(),), -torch.inf),
|
||||
)
|
||||
)
|
||||
assert torch.equal(param, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_shape,runtime_shape,error",
|
||||
[
|
||||
((2, 4), (4,), RuntimeError),
|
||||
((8,), (2, 2), AssertionError),
|
||||
((7,), (4,), ValueError),
|
||||
((8,), (3,), AssertionError),
|
||||
],
|
||||
)
|
||||
def test_attention_sink_loader_rejects_invalid_shapes(
|
||||
monkeypatch, checkpoint_shape, runtime_shape, error
|
||||
):
|
||||
"""Invalid sink shapes fail before changing the runtime parameter."""
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
param = torch.nn.Parameter(torch.ones(runtime_shape), requires_grad=False)
|
||||
with pytest.raises(error):
|
||||
load_attention_sink(param, torch.zeros(checkpoint_shape))
|
||||
assert torch.equal(param, torch.ones(runtime_shape))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param_dtype", [torch.bfloat16, torch.float32])
|
||||
def test_attention_sink_loader_keeps_param_dtype(monkeypatch, param_dtype):
|
||||
"""A checkpoint dtype different from the runtime parameter is cast, not copied."""
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
param = torch.nn.Parameter(torch.zeros(4, dtype=param_dtype), requires_grad=False)
|
||||
load_attention_sink(param, torch.arange(4, dtype=torch.float32))
|
||||
assert param.dtype == param_dtype
|
||||
assert torch.equal(param.float(), torch.arange(4, dtype=torch.float32))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a GPU")
|
||||
def test_attention_sink_loader_accepts_cpu_checkpoint(monkeypatch):
|
||||
"""Sinks are normally loaded from CPU checkpoints into device parameters."""
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 1)
|
||||
param = torch.nn.Parameter(
|
||||
torch.full((6,), 123.0, device="cuda"), requires_grad=False
|
||||
)
|
||||
# The test suite sets the default device, so pin the checkpoint to CPU.
|
||||
checkpoint = torch.arange(8, dtype=torch.float32, device="cpu")
|
||||
load_attention_sink(param, checkpoint)
|
||||
assert param.device.type == "cuda"
|
||||
expected = torch.arange(4, 8, dtype=torch.float32, device="cpu")
|
||||
assert torch.equal(param[:4].cpu(), expected)
|
||||
assert torch.isinf(param[4:]).all() and (param[4:] < 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads,dtype", [(4, torch.float32), (8, torch.bfloat16), (6, None)]
|
||||
)
|
||||
def test_make_attention_sink_registers_loader_and_padding(num_heads, dtype):
|
||||
"""The factory is the single place that creates and registers a sink."""
|
||||
sink = weight_utils.make_attention_sink(num_heads, dtype)
|
||||
assert sink.shape == (num_heads,)
|
||||
assert sink.dtype == (dtype or torch.get_default_dtype())
|
||||
assert sink.requires_grad is False
|
||||
assert torch.isinf(sink).all() and (sink < 0).all()
|
||||
assert sink.weight_loader is load_attention_sink
|
||||
|
||||
|
||||
def test_download_weights_from_hf():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# assert LocalEntryNotFoundError error is thrown
|
||||
|
||||
Reference in New Issue
Block a user