[Core][LoRA] Support fp32 lm_head (head_dtype) on the LoRA path (#48525)
Signed-off-by: Karthik Kothuri <karthikkothuri2009@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
6e073440b1
commit
4f7fffb92f
@@ -368,8 +368,9 @@ def test_embeddings(
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
@pytest.mark.parametrize("vocab_size", [64000, 256512, 258048])
|
||||
@pytest.mark.parametrize("stage", STAGES)
|
||||
@pytest.mark.parametrize("head_dtype", [None, torch.float32])
|
||||
def test_lm_head_logits_processor(
|
||||
default_vllm_config, dist_init, num_loras, device, vocab_size, stage
|
||||
default_vllm_config, dist_init, num_loras, device, vocab_size, stage, head_dtype
|
||||
) -> None:
|
||||
if current_platform.is_cuda_alike() or current_platform.is_xpu():
|
||||
torch.accelerator.set_device_index(device)
|
||||
@@ -391,6 +392,8 @@ def test_lm_head_logits_processor(
|
||||
linear.weight.data = torch.rand_like(linear.weight.data)
|
||||
linear.weight.data[:, vocab_size:] = 0
|
||||
logits_processor = LogitsProcessor(vocab_size)
|
||||
# Exercise an fp32 lm_head (head_dtype) on the LoRA path.
|
||||
logits_processor.head_dtype = head_dtype
|
||||
lora_logits_processor = LogitsProcessorWithLoRA(
|
||||
logits_processor, 1024, linear.weight.dtype, linear.weight.device, None
|
||||
)
|
||||
|
||||
@@ -136,22 +136,6 @@ def test_get_top_tokens_honors_head_dtype(default_vllm_config):
|
||||
assert torch.equal(top, expected)
|
||||
|
||||
|
||||
def test_fp32_head_rejected_with_lora(default_vllm_config):
|
||||
from vllm.lora.layers.logits_processor import LogitsProcessorWithLoRA
|
||||
|
||||
base = _build_processor(64)
|
||||
base.head_dtype = torch.float32
|
||||
|
||||
with pytest.raises(ValueError, match="not yet supported with LoRA"):
|
||||
LogitsProcessorWithLoRA(
|
||||
base,
|
||||
hidden_size=16,
|
||||
dtype=torch.bfloat16,
|
||||
device=torch.device("cpu"),
|
||||
sharded_to_full_mapping=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_fp32_head_e2e_no_nan():
|
||||
"""An fp32 head produces finite logprobs end-to-end.
|
||||
|
||||
Reference in New Issue
Block a user