[CPU][Whisper] Support W4A16 quantized Whisper on the CPU WNA16 kernel (#58268)
Signed-off-by: Harshal Adhav <harshal.adhav@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Cursor
Li, Jiang <jiang1.li@intel.com>
parent
fcf4750466
commit
5faf81a429
@@ -5,6 +5,7 @@ from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSpeechSeq2Seq
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
@@ -264,6 +265,49 @@ def test_parse_language_detection_output():
|
||||
cls.parse_language_detection_output([], make_tokenizer("anything"))
|
||||
|
||||
|
||||
def test_create_fake_bias_for_k_proj_injects_out_features():
|
||||
"""Packed k_proj weights get a zeros bias of length ``out_features``."""
|
||||
from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj
|
||||
|
||||
packed = torch.ones(2, 16)
|
||||
result = dict(
|
||||
_create_fake_bias_for_k_proj(
|
||||
[("model.encoder.layers.0.self_attn.k_proj.weight_packed", packed)],
|
||||
".k_proj.weight_packed",
|
||||
out_features=8,
|
||||
)
|
||||
)
|
||||
bias_name = "model.encoder.layers.0.self_attn.k_proj.bias"
|
||||
assert bias_name in result
|
||||
assert "model.encoder.layers.0.self_attn.k_proj.bias_packed" not in result
|
||||
assert tuple(result[bias_name].shape) == (8,)
|
||||
assert torch.equal(result[bias_name], torch.zeros(8))
|
||||
|
||||
|
||||
def test_create_fake_bias_for_k_proj_forwards_real_bias():
|
||||
"""A checkpoint bias is forwarded and no extra fake bias is emitted."""
|
||||
from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj
|
||||
|
||||
weight = torch.ones(8, 16)
|
||||
real_bias = torch.ones(8)
|
||||
items = list(
|
||||
_create_fake_bias_for_k_proj(
|
||||
[
|
||||
("layers.0.self_attn.k_proj.weight", weight),
|
||||
("layers.0.self_attn.k_proj.bias", real_bias),
|
||||
],
|
||||
".k_proj.weight",
|
||||
out_features=8,
|
||||
)
|
||||
)
|
||||
names = [n for n, _ in items]
|
||||
assert names == [
|
||||
"layers.0.self_attn.k_proj.weight",
|
||||
"layers.0.self_attn.k_proj.bias",
|
||||
]
|
||||
assert items[1][1] is real_bias
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu_model
|
||||
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
|
||||
|
||||
Reference in New Issue
Block a user