[V0 deprecation] Deprecate V0 Neuron backend (#21159)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
@@ -287,15 +287,6 @@ def test_prefix_cache_default():
|
||||
},
|
||||
"mm-processor-kwargs"
|
||||
),
|
||||
(
|
||||
'{"cast_logits_dtype":"bfloat16","sequence_parallel_norm":true,"sequence_parallel_norm_threshold":2048}',
|
||||
{
|
||||
"cast_logits_dtype": "bfloat16",
|
||||
"sequence_parallel_norm": True,
|
||||
"sequence_parallel_norm_threshold": 2048,
|
||||
},
|
||||
"override-neuron-config"
|
||||
),
|
||||
])
|
||||
# yapf: enable
|
||||
def test_composite_arg_parser(arg, expected, option):
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.layers.activation import FastGELU, SiluAndMul
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize("activation", ["silu_and_mul", "gelu_fast"])
|
||||
@pytest.mark.parametrize("num_tokens,d,dtype", [
|
||||
(7, 512, torch.half),
|
||||
(7, 512, torch.float),
|
||||
(83, 512, torch.half),
|
||||
])
|
||||
@torch.inference_mode()
|
||||
def test_act_and_mul(
|
||||
activation: str,
|
||||
num_tokens: int,
|
||||
d: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
import torch_xla.core.xla_model as xm
|
||||
|
||||
device = xm.xla_device()
|
||||
current_platform.seed_everything(0)
|
||||
torch.set_default_device("cpu")
|
||||
x = torch.randn(num_tokens, 2 * d, dtype=dtype).to(device=device)
|
||||
if activation == "silu_and_mul":
|
||||
layer = SiluAndMul()
|
||||
fn = layer.forward_native
|
||||
elif activation == "gelu_fast":
|
||||
layer = FastGELU()
|
||||
fn = F.gelu
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"activation {activation} is not implemented.")
|
||||
assert x.is_xla, "input tensor under testing is expected to be XLA tensor."
|
||||
out = layer.to(device=device).forward_neuron(x)
|
||||
ref_out = fn(x.cpu())
|
||||
torch.testing.assert_close(out.cpu(), ref_out, atol=0.01, rtol=0.0)
|
||||
@@ -1,154 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import neuronxcc.nki.language as nl
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from neuronxcc import nki
|
||||
|
||||
from vllm.attention.ops.nki_flash_attn import (
|
||||
load_block_tables, transform_block_tables_for_indirect_load)
|
||||
|
||||
|
||||
def is_power_of_2(n):
|
||||
return n > 0 and (n & (n - 1) == 0)
|
||||
|
||||
|
||||
def nki_load_and_transform_block_tables(
|
||||
block_tables,
|
||||
num_tiles,
|
||||
num_blocks_per_tile,
|
||||
num_head,
|
||||
head_id,
|
||||
block_size_tiling_factor,
|
||||
):
|
||||
assert is_power_of_2(
|
||||
num_blocks_per_tile), f"{num_blocks_per_tile=} must be power of 2"
|
||||
block_tables_sbuf = load_block_tables(block_tables, num_tiles,
|
||||
num_blocks_per_tile)
|
||||
|
||||
# we need to pass an Index as head_id
|
||||
head_id = nl.arange(1)[None, :] + head_id
|
||||
|
||||
block_tables_transposed = transform_block_tables_for_indirect_load(
|
||||
block_tables_sbuf, block_size_tiling_factor, num_head, head_id)
|
||||
B_P_SIZE = 128
|
||||
assert block_tables_transposed.shape[1] == B_P_SIZE
|
||||
|
||||
out = nl.ndarray(
|
||||
block_tables_transposed.shape,
|
||||
dtype=nl.int32,
|
||||
buffer=nl.shared_hbm,
|
||||
)
|
||||
for i in nl.affine_range(block_tables_transposed.shape[0]):
|
||||
nl.store(dst=out[i], value=block_tables_transposed[i])
|
||||
return out
|
||||
|
||||
|
||||
def ref_block_tables_transform(
|
||||
block_tables,
|
||||
num_tiles,
|
||||
num_blocks_per_tile,
|
||||
num_head,
|
||||
head_id,
|
||||
block_size_tiling_factor,
|
||||
):
|
||||
assert block_tables.numel() == num_tiles * num_blocks_per_tile
|
||||
block_tables = block_tables.view(num_tiles, num_blocks_per_tile)
|
||||
B_F_SIZE = 128
|
||||
num_tiles_padded = (num_tiles + B_F_SIZE - 1) // B_F_SIZE * B_F_SIZE
|
||||
block_tables = F.pad(
|
||||
block_tables,
|
||||
(0, 0, 0, num_tiles_padded - num_tiles),
|
||||
"constant",
|
||||
0,
|
||||
)
|
||||
|
||||
block_tables = block_tables * num_head + head_id
|
||||
block_tables = block_tables.view(num_tiles_padded, num_blocks_per_tile, 1)
|
||||
offset = torch.arange(0, block_size_tiling_factor).view(1, 1, -1)
|
||||
block_tables = block_tables * block_size_tiling_factor + offset
|
||||
block_tables_transposed = block_tables.view(num_tiles_padded, -1).t()
|
||||
|
||||
num_blocks_per_tile = block_tables_transposed.shape[0]
|
||||
assert num_blocks_per_tile % B_F_SIZE == 0
|
||||
return block_tables_transposed.view(num_blocks_per_tile // B_F_SIZE,
|
||||
B_F_SIZE, num_tiles_padded)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"q_head_per_kv_head,head_id",
|
||||
[
|
||||
(1, 0),
|
||||
(3, 1),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_tiles,num_blocks_per_tile",
|
||||
[
|
||||
(1, 1),
|
||||
(13, 16),
|
||||
(17, 128),
|
||||
(35, 512),
|
||||
(128, 128),
|
||||
(130, 64),
|
||||
(280, 256),
|
||||
(315, 1),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_load_and_transform_block_tables(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
num_tiles,
|
||||
num_blocks_per_tile,
|
||||
q_head_per_kv_head,
|
||||
head_id,
|
||||
) -> None:
|
||||
import torch_xla.core.xla_model as xm
|
||||
|
||||
device = xm.xla_device()
|
||||
|
||||
compiler_flags_str = " ".join([
|
||||
"-O1",
|
||||
"--retry_failed_compilation",
|
||||
])
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("NEURON_CC_FLAGS", compiler_flags_str)
|
||||
|
||||
torch.manual_seed(10000)
|
||||
torch.set_printoptions(sci_mode=False)
|
||||
|
||||
# On Neuron, we need B_P_SIZE = 128 blocks to make DMA efficient
|
||||
B_P_SIZE = 128
|
||||
if num_blocks_per_tile < B_P_SIZE:
|
||||
assert B_P_SIZE % num_blocks_per_tile == 0
|
||||
block_size_tiling_factor = B_P_SIZE // num_blocks_per_tile
|
||||
else:
|
||||
block_size_tiling_factor = 1
|
||||
max_num_blocks = 100000
|
||||
block_tables = torch.randint(
|
||||
0,
|
||||
max_num_blocks,
|
||||
(num_tiles * num_blocks_per_tile, ),
|
||||
dtype=torch.int32,
|
||||
)
|
||||
nki_out = nki.jit(nki_load_and_transform_block_tables)[1, 1](
|
||||
block_tables.to(device=device),
|
||||
num_tiles,
|
||||
num_blocks_per_tile,
|
||||
q_head_per_kv_head,
|
||||
head_id,
|
||||
block_size_tiling_factor,
|
||||
).cpu()
|
||||
ref_out = ref_block_tables_transform(
|
||||
block_tables,
|
||||
num_tiles,
|
||||
num_blocks_per_tile,
|
||||
q_head_per_kv_head,
|
||||
head_id,
|
||||
block_size_tiling_factor,
|
||||
)
|
||||
assert (nki_out.shape == ref_out.shape
|
||||
), f"{nki_out.shape=} != {ref_out.shape=}"
|
||||
assert torch.all(nki_out == ref_out)
|
||||
@@ -1,86 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.attention.ops.nki_flash_attn import reshape_and_cache
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens, n_kv_head, d_head, num_blocks, block_size",
|
||||
[
|
||||
# Small model configuration (e.g., GPT-2 small)
|
||||
(32, 12, 64, 4, 128), # Typical sequence processing
|
||||
(1, 12, 64, 4, 128), # Single token update
|
||||
(128, 12, 64, 4, 128), # Longer sequence
|
||||
|
||||
# Medium model configuration (e.g., GPT-2 medium)
|
||||
(64, 16, 96, 8, 256), # Standard batch
|
||||
(256, 16, 96, 8, 256), # Large batch
|
||||
|
||||
# Large model configuration (e.g., GPT-3 style)
|
||||
(48, 32, 128, 16, 512), # Typical processing window
|
||||
(512, 32, 128, 16, 512), # Full context window
|
||||
|
||||
# Edge cases and stress tests
|
||||
(1024, 8, 32, 32, 32), # Many tokens, small heads
|
||||
(16, 64, 256, 4, 64), # Few tokens, many heads
|
||||
(2048, 24, 128, 64, 128), # Large scale test
|
||||
|
||||
# Minimal configurations for debugging
|
||||
(4, 2, 16, 2, 16), # Tiny test case
|
||||
(1, 1, 8, 1, 8), # Minimal possible
|
||||
])
|
||||
def test_reshape_and_cache(num_tokens, n_kv_head, d_head, num_blocks,
|
||||
block_size):
|
||||
# Set random seed for reproducibility
|
||||
torch.manual_seed(42)
|
||||
|
||||
# Create CPU tensors for reference implementation
|
||||
key_cpu = torch.randn(num_tokens, n_kv_head, d_head) / torch.sqrt(
|
||||
torch.tensor(d_head))
|
||||
value_cpu = torch.randn(num_tokens, n_kv_head, d_head) / torch.sqrt(
|
||||
torch.tensor(d_head))
|
||||
key_cache_cpu = torch.zeros(num_blocks, n_kv_head, block_size, d_head)
|
||||
value_cache_cpu = torch.zeros(num_blocks, n_kv_head, block_size, d_head)
|
||||
slot_mapping_cpu = torch.randperm(num_blocks * block_size)[:num_tokens]
|
||||
|
||||
# Run reference implementation on CPU
|
||||
block_indices = torch.div(slot_mapping_cpu,
|
||||
block_size,
|
||||
rounding_mode="floor")
|
||||
block_offsets = slot_mapping_cpu % block_size
|
||||
|
||||
for i in range(num_tokens):
|
||||
block_idx = block_indices[i]
|
||||
block_offset = block_offsets[i]
|
||||
key_cache_cpu[block_idx, :, block_offset, :] = key_cpu[i]
|
||||
value_cache_cpu[block_idx, :, block_offset, :] = value_cpu[i]
|
||||
|
||||
# Create XLA device tensors
|
||||
device = torch.device('xla')
|
||||
key = key_cpu.to(device)
|
||||
value = value_cpu.to(device)
|
||||
key_cache = torch.zeros_like(key_cache_cpu, device=device)
|
||||
value_cache = torch.zeros_like(value_cache_cpu, device=device)
|
||||
slot_mapping = slot_mapping_cpu.to(device)
|
||||
kv_cache = torch.stack([key_cache, value_cache])
|
||||
|
||||
# Run vectorized implementation on XLA device
|
||||
reshape_and_cache(key, value, kv_cache, slot_mapping)
|
||||
key_cache, value_cache = torch.unbind(kv_cache, dim=0)
|
||||
|
||||
# Move results back to CPU for comparison
|
||||
key_cache_result = key_cache.cpu()
|
||||
value_cache_result = value_cache.cpu()
|
||||
|
||||
# Assert results match
|
||||
torch.testing.assert_close(key_cache_result,
|
||||
key_cache_cpu,
|
||||
rtol=1e-5,
|
||||
atol=1e-5)
|
||||
torch.testing.assert_close(value_cache_result,
|
||||
value_cache_cpu,
|
||||
rtol=1e-5,
|
||||
atol=1e-5)
|
||||
@@ -1,57 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens,hidden_size,add_residual,dtype", [
|
||||
(7, 8, False, torch.half),
|
||||
(83, 768, False, torch.half),
|
||||
(83, 768, True, torch.half),
|
||||
(83, 768, True, torch.bfloat16),
|
||||
(83, 768, True, torch.float32),
|
||||
])
|
||||
@torch.inference_mode()
|
||||
def test_rms_norm(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
add_residual: bool,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
import torch_xla.core.xla_model as xm
|
||||
|
||||
device = xm.xla_device()
|
||||
current_platform.seed_everything(0)
|
||||
torch.set_default_device("cpu")
|
||||
layer = RMSNorm(hidden_size).to(dtype=dtype)
|
||||
layer.weight.data.normal_(mean=1.0, std=0.1)
|
||||
scale = 1 / (2 * hidden_size)
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype).to(device=device)
|
||||
x *= scale
|
||||
residual = torch.randn_like(x) * scale if add_residual else None
|
||||
|
||||
residual_cpu = residual.cpu() if add_residual else None
|
||||
ref_out = layer.to(device="cpu").forward_native(x.cpu(), residual_cpu)
|
||||
assert x.is_xla, "input tensor under testing is expected to be XLA tensor."
|
||||
out = layer.to(device=device)(x, residual)
|
||||
|
||||
# NOTE(woosuk): LayerNorm operators (including RMS) typically have larger
|
||||
# numerical errors than other operators because they involve reductions.
|
||||
# Therefore, we use a larger tolerance.
|
||||
if add_residual:
|
||||
assert out[0].is_xla, "output tensor is expected to be XLA tensor"
|
||||
torch.testing.assert_close(out[0].cpu(),
|
||||
ref_out[0],
|
||||
atol=1e-2,
|
||||
rtol=1e-2)
|
||||
torch.testing.assert_close(out[1].cpu(),
|
||||
ref_out[1],
|
||||
atol=1e-2,
|
||||
rtol=1e-2)
|
||||
else:
|
||||
assert out.is_xla, "output tensor is expected to be XLA tensor"
|
||||
torch.testing.assert_close(out.cpu(), ref_out, atol=1e-2, rtol=1e-2)
|
||||
@@ -1,95 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import random
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.model_executor.utils import set_random_seed
|
||||
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
|
||||
from vllm.utils import is_pin_memory_available
|
||||
|
||||
|
||||
class MockLogitsProcessor(LogitsProcessor):
|
||||
|
||||
def __init__(self, vocab_size: int, scale: float,
|
||||
fake_logits: torch.Tensor):
|
||||
super().__init__(vocab_size=vocab_size, scale=scale)
|
||||
self.fake_logits = fake_logits.clone()
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
with patch(
|
||||
"vllm.model_executor.layers.logits_processor._prune_hidden_states",
|
||||
lambda x, y: x
|
||||
), patch(
|
||||
"vllm.model_executor.layers.logits_processor.LogitsProcessor._get_logits",
|
||||
lambda *args, **kwargs: self.fake_logits):
|
||||
return super().forward(*args, **kwargs)
|
||||
|
||||
|
||||
def _prepare_test(
|
||||
batch_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor, MockLogitsProcessor]:
|
||||
vocab_size = 32000
|
||||
input_tensor = torch.rand((batch_size, 1024), dtype=torch.float16)
|
||||
fake_logits = torch.full((batch_size, vocab_size),
|
||||
1e-2,
|
||||
dtype=input_tensor.dtype)
|
||||
logits_processor = MockLogitsProcessor(32000, 0.5, fake_logits)
|
||||
return input_tensor, fake_logits, logits_processor
|
||||
|
||||
|
||||
RANDOM_SEEDS = list(range(8))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seed", RANDOM_SEEDS)
|
||||
def test_logits_processors(seed: int):
|
||||
import torch_xla.core.xla_model as xm
|
||||
|
||||
device = xm.xla_device()
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device("cpu")
|
||||
batch_size = random.randint(1, 256)
|
||||
input_tensor, fake_logits, logits_processor = _prepare_test(batch_size)
|
||||
|
||||
# This sample logits processor gives infinite score to the i-th token,
|
||||
# where i is the length of the input sequence.
|
||||
# We therefore expect the output token sequence to be [0, 1, 2, ...]
|
||||
def pick_ith(token_ids, logits):
|
||||
logits[len(token_ids)] = float("inf")
|
||||
return logits
|
||||
|
||||
seq_group_metadata_list = []
|
||||
seq_lens = []
|
||||
for i in range(batch_size):
|
||||
seq_group_metadata_list.append(
|
||||
SequenceGroupMetadata(
|
||||
request_id=f"test_{i}",
|
||||
is_prompt=True,
|
||||
seq_data={0: SequenceData.from_seqs([1, 2, 3])},
|
||||
sampling_params=SamplingParams(temperature=0,
|
||||
logits_processors=[pick_ith]),
|
||||
block_tables={0: [1]},
|
||||
))
|
||||
seq_lens.append(seq_group_metadata_list[-1].seq_data[0].get_len())
|
||||
|
||||
sampling_metadata = SamplingMetadata.prepare(
|
||||
seq_group_metadata_list,
|
||||
seq_lens,
|
||||
query_lens=seq_lens,
|
||||
device=device,
|
||||
pin_memory=is_pin_memory_available())
|
||||
logits_processor_output = logits_processor(
|
||||
lm_head=None,
|
||||
hidden_states=input_tensor,
|
||||
sampling_metadata=sampling_metadata)
|
||||
|
||||
fake_logits *= logits_processor.scale
|
||||
torch.testing.assert_close(logits_processor_output[:, 1],
|
||||
fake_logits[:, 1],
|
||||
rtol=1e-4,
|
||||
atol=0.0)
|
||||
@@ -1,127 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.neuron import NeuronFramework
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import SequenceData, SequenceGroupMetadata
|
||||
from vllm.worker.neuron_model_runner import NeuronModelRunner
|
||||
|
||||
os.environ[
|
||||
'VLLM_NEURON_FRAMEWORK'] = NeuronFramework.TRANSFORMERS_NEURONX.value
|
||||
|
||||
|
||||
def _create_neuron_model_runner(model: str, *args,
|
||||
**kwargs) -> NeuronModelRunner:
|
||||
engine_args = EngineArgs(model, *args, **kwargs)
|
||||
engine_config = engine_args.create_engine_config()
|
||||
vllm_config = VllmConfig(
|
||||
model_config=engine_config.model_config,
|
||||
parallel_config=engine_config.parallel_config,
|
||||
scheduler_config=engine_config.scheduler_config,
|
||||
device_config=engine_config.device_config,
|
||||
)
|
||||
neuron_model_runner = NeuronModelRunner(vllm_config=vllm_config)
|
||||
return neuron_model_runner
|
||||
|
||||
|
||||
def test_update_neuron_sampling_params_not_full_batch():
|
||||
os.environ["NEURON_ON_DEVICE_SAMPLING_DISABLED"] = "0"
|
||||
model_runner = _create_neuron_model_runner(
|
||||
"facebook/opt-125m",
|
||||
seed=0,
|
||||
dtype="float16",
|
||||
max_num_seqs=2,
|
||||
)
|
||||
assert not model_runner._on_device_sampling_disabled
|
||||
# Test sampling param updating only when TNx is framework
|
||||
# NxDI handles sampling parameter updating inside model
|
||||
if current_platform.use_transformers_neuronx():
|
||||
model_mock = MagicMock()
|
||||
model_runner.model = model_mock
|
||||
|
||||
seq_group_metadata_list = [
|
||||
SequenceGroupMetadata(
|
||||
request_id="test_0",
|
||||
is_prompt=True,
|
||||
seq_data={0: SequenceData.from_seqs([1, 2, 3])},
|
||||
sampling_params=SamplingParams(temperature=0.5,
|
||||
top_k=1,
|
||||
top_p=0.5),
|
||||
block_tables={0: [1]},
|
||||
)
|
||||
]
|
||||
|
||||
model_runner.prepare_model_input(seq_group_metadata_list)
|
||||
|
||||
# Index neuron sampling parameters based on block_tables indices.
|
||||
# The first block_id of the sequence 0 is 1, so its parameters are
|
||||
# placed at index 1. So the sampling parameters will be:
|
||||
# Index 0: default sampling parameters
|
||||
# Index 1: sequecne 0's sampling parameters.
|
||||
neuron_sampling_params = (
|
||||
model_runner.model_config.neuron_sampling_params)
|
||||
assert neuron_sampling_params.temperature == [1.0, 0.5]
|
||||
assert neuron_sampling_params.top_k == [
|
||||
model_runner._MAX_NEURON_SAMPLING_TOP_K, 1
|
||||
]
|
||||
assert neuron_sampling_params.top_p == [1.0, 0.5]
|
||||
model_mock.model.update_generation_config.assert_called_once_with(
|
||||
neuron_sampling_params)
|
||||
|
||||
|
||||
def test_update_neuron_sampling_params_full_batch():
|
||||
os.environ["NEURON_ON_DEVICE_SAMPLING_DISABLED"] = "0"
|
||||
model_runner = _create_neuron_model_runner(
|
||||
"facebook/opt-125m",
|
||||
seed=0,
|
||||
dtype="float16",
|
||||
max_num_seqs=2,
|
||||
)
|
||||
assert not model_runner._on_device_sampling_disabled
|
||||
|
||||
# Test sampling param updating only when TNx is framework
|
||||
# NxDI handles sampling parameter updating inside model
|
||||
if current_platform.use_transformers_neuronx():
|
||||
model_mock = MagicMock()
|
||||
model_runner.model = model_mock
|
||||
|
||||
seq_group_metadata_list = [
|
||||
SequenceGroupMetadata(
|
||||
request_id="test_0",
|
||||
is_prompt=True,
|
||||
seq_data={0: SequenceData.from_seqs([1, 2, 3])},
|
||||
sampling_params=SamplingParams(temperature=0.5,
|
||||
top_k=1,
|
||||
top_p=0.5),
|
||||
block_tables={0: [1]},
|
||||
),
|
||||
SequenceGroupMetadata(
|
||||
request_id="test_0",
|
||||
is_prompt=True,
|
||||
seq_data={1: SequenceData.from_seqs([4, 5, 6])},
|
||||
sampling_params=SamplingParams(temperature=0.2,
|
||||
top_k=2,
|
||||
top_p=0.2),
|
||||
block_tables={1: [0]},
|
||||
)
|
||||
]
|
||||
|
||||
model_runner.prepare_model_input(seq_group_metadata_list)
|
||||
|
||||
# Index neuron sampling parameters based on block_tables indices.
|
||||
# The first block_id of the sequence 0 is 1, so its parameters are
|
||||
# placed at index 1. So the sampling parameters will be:
|
||||
# Index 0: sequence 1's sampling parameters
|
||||
# Index 1: sequecne 0's sampling parameters.
|
||||
neuron_sampling_params = (
|
||||
model_runner.model_config.neuron_sampling_params)
|
||||
assert neuron_sampling_params.temperature == [0.2, 0.5]
|
||||
assert neuron_sampling_params.top_k == [2, 1]
|
||||
assert neuron_sampling_params.top_p == [0.2, 0.5]
|
||||
model_mock.model.update_generation_config.assert_called_once_with(
|
||||
neuron_sampling_params)
|
||||
@@ -1,12 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.model_executor.layers.quantization.neuron_quant import (
|
||||
NeuronQuantConfig)
|
||||
|
||||
|
||||
def test_get_supported_act_dtypes():
|
||||
neuron_quant_config = NeuronQuantConfig()
|
||||
supported_act_dtypes = neuron_quant_config.get_supported_act_dtypes()
|
||||
target_list = ["any_dtype1", "any_dtype2"]
|
||||
for dtype in target_list:
|
||||
assert dtype in supported_act_dtypes
|
||||
@@ -1,514 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.utils import cdiv
|
||||
|
||||
|
||||
class BlockDiagonalCausalFromBottomRightMask:
|
||||
|
||||
@staticmethod
|
||||
def _from_seqlens(query_lens, seq_lens, block_size=None):
|
||||
from torch import logical_and, logical_or
|
||||
|
||||
contexted = block_size is None
|
||||
context_lens = torch.tensor(seq_lens) - torch.tensor(query_lens)
|
||||
n_queries = sum(query_lens)
|
||||
num_seqs = len(query_lens)
|
||||
if contexted:
|
||||
key_lens_blockaligned = seq_lens
|
||||
else:
|
||||
n_blocks_per_seq = (context_lens + block_size - 1) // block_size
|
||||
offset_per_seq = n_blocks_per_seq * block_size
|
||||
key_lens_blockaligned = offset_per_seq[:num_seqs].tolist()
|
||||
n_keys = sum(key_lens_blockaligned)
|
||||
|
||||
a = (torch.arange(n_queries).reshape(n_queries,
|
||||
1).expand(n_queries, n_keys))
|
||||
b = torch.arange(n_keys).reshape(1, n_keys).expand(n_queries, n_keys)
|
||||
q_cumsum = torch.tensor([0] + query_lens).cumsum(dim=0)
|
||||
k_cumsum = torch.tensor([0] + key_lens_blockaligned).cumsum(dim=0)
|
||||
|
||||
prior_mask = torch.zeros(n_queries, n_keys)
|
||||
new_masks: list[torch.Tensor] = []
|
||||
for seq_id in range(num_seqs):
|
||||
ri = q_cumsum[seq_id]
|
||||
ci = k_cumsum[seq_id]
|
||||
nr = query_lens[seq_id]
|
||||
|
||||
if contexted:
|
||||
nc = seq_lens[seq_id]
|
||||
a_offset = ci + nc - ri - nr
|
||||
new_mask = (a + a_offset) >= b
|
||||
else:
|
||||
nc = context_lens[seq_id]
|
||||
a_offset = ci + nc - 1
|
||||
new_mask = a_offset >= b
|
||||
|
||||
left_mask = b >= ci
|
||||
top_mask = a >= ri
|
||||
bottom_mask = a < (ri + nr)
|
||||
|
||||
new_mask = logical_and(
|
||||
logical_and(logical_and(new_mask, left_mask), top_mask),
|
||||
bottom_mask,
|
||||
)
|
||||
prior_mask = logical_or(prior_mask, new_mask)
|
||||
new_masks = new_masks + [new_mask]
|
||||
return prior_mask
|
||||
|
||||
@staticmethod
|
||||
def from_seqlens(query_lens, seq_lens, block_size=None):
|
||||
contexted = block_size is None
|
||||
if contexted:
|
||||
prior_mask = BlockDiagonalCausalFromBottomRightMask._from_seqlens(
|
||||
query_lens, seq_lens)
|
||||
active_mask = None
|
||||
else:
|
||||
prior_mask = BlockDiagonalCausalFromBottomRightMask._from_seqlens(
|
||||
query_lens, seq_lens, block_size)
|
||||
active_mask = BlockDiagonalCausalFromBottomRightMask._from_seqlens(
|
||||
query_lens, query_lens)
|
||||
return prior_mask, active_mask
|
||||
|
||||
|
||||
def ref_softmax(x: torch.Tensor,
|
||||
dim: int,
|
||||
mixed_precision=False,
|
||||
return_max_reduce=False):
|
||||
max_value = torch.amax(x, dim=dim, keepdims=True)
|
||||
exp = torch.exp(x - max_value)
|
||||
if mixed_precision:
|
||||
sum_value = torch.sum(exp.astype(torch.float32),
|
||||
dim=dim,
|
||||
keepdims=True).astype(x.dtype)
|
||||
else:
|
||||
sum_value = torch.sum(exp, dim=dim, keepdims=True)
|
||||
if return_max_reduce:
|
||||
return exp / sum_value, max_value, torch.reciprocal(sum_value)
|
||||
return exp / sum_value
|
||||
|
||||
|
||||
def ref_masked_attention(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
scale: float,
|
||||
attn_mask: Optional[torch.Tensor] = None,
|
||||
return_max_reduce: Optional[bool] = False,
|
||||
) -> torch.Tensor:
|
||||
scaled_qk = scale * torch.einsum("qhd,khd->hqk", query, key).float()
|
||||
if attn_mask is not None:
|
||||
masked_score = scaled_qk + attn_mask.float()
|
||||
if return_max_reduce:
|
||||
norm_score, cached_max, cached_sum_reciprocal = ref_softmax(
|
||||
masked_score, dim=-1, return_max_reduce=True)
|
||||
else:
|
||||
norm_score = ref_softmax(masked_score, dim=-1)
|
||||
out = torch.einsum("hqk,khd->qhd", norm_score.to(value.dtype), value)
|
||||
if return_max_reduce:
|
||||
return (
|
||||
out,
|
||||
cached_max,
|
||||
cached_sum_reciprocal,
|
||||
norm_score,
|
||||
masked_score,
|
||||
scaled_qk,
|
||||
)
|
||||
else:
|
||||
return (out, )
|
||||
|
||||
|
||||
def ref_context_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
query_lens,
|
||||
seq_lens,
|
||||
head_size,
|
||||
num_queries_per_kv,
|
||||
return_max_reduce=False,
|
||||
):
|
||||
scale = float(1.0 / (head_size**0.5))
|
||||
if num_queries_per_kv > 1:
|
||||
# Handle MQA and GQA
|
||||
key = torch.repeat_interleave(key, num_queries_per_kv, dim=1)
|
||||
value = torch.repeat_interleave(value, num_queries_per_kv, dim=1)
|
||||
|
||||
attn_mask, _ = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
|
||||
query_lens, seq_lens)
|
||||
|
||||
# convert binary mask to -inf values
|
||||
attn_mask = torch.logical_not(attn_mask)
|
||||
attn_mask = attn_mask.float() * -30000
|
||||
|
||||
output, *debug_tensors = ref_masked_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
scale,
|
||||
attn_mask,
|
||||
return_max_reduce=return_max_reduce,
|
||||
)
|
||||
|
||||
output = output.unsqueeze(1)
|
||||
if return_max_reduce:
|
||||
cached_max, cached_sum_reciprocal, lse, masked_score, scaled_qk = (
|
||||
debug_tensors)
|
||||
return (
|
||||
output,
|
||||
cached_max,
|
||||
cached_sum_reciprocal,
|
||||
lse,
|
||||
masked_score,
|
||||
scaled_qk,
|
||||
)
|
||||
else:
|
||||
return output
|
||||
|
||||
|
||||
def sample_inputs(
|
||||
prefill_batch_size,
|
||||
decode_batch_size,
|
||||
min_query_len,
|
||||
max_query_len,
|
||||
min_ctx_len,
|
||||
max_ctx_len,
|
||||
block_size,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype,
|
||||
):
|
||||
batch_size = prefill_batch_size + decode_batch_size
|
||||
max_model_len = (max_query_len + max_ctx_len) * 4
|
||||
max_block_per_request = max_model_len // block_size
|
||||
cache_size = (batch_size * max_block_per_request) + 2
|
||||
prefill_ctx_lens = torch.randint(min_ctx_len,
|
||||
max_ctx_len + 1, (prefill_batch_size, ),
|
||||
dtype=torch.long).tolist()
|
||||
decode_ctx_lens = torch.randint(min_ctx_len,
|
||||
max_ctx_len + 1, (decode_batch_size, ),
|
||||
dtype=torch.long).tolist()
|
||||
ctx_lens = prefill_ctx_lens + decode_ctx_lens
|
||||
query_lens = torch.randint(
|
||||
min_query_len,
|
||||
max_query_len + 1,
|
||||
(prefill_batch_size, ),
|
||||
dtype=torch.long,
|
||||
).tolist() + [1 for _ in range(decode_batch_size)]
|
||||
seq_lens = [a + b for a, b in zip(query_lens, ctx_lens)]
|
||||
|
||||
num_tokens = sum(query_lens)
|
||||
query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
|
||||
query.uniform_(-1, 1)
|
||||
torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
|
||||
|
||||
kv = torch.empty(sum(seq_lens), 2, num_kv_heads, head_size, dtype=dtype)
|
||||
kv.uniform_(-1, 1)
|
||||
key, value = kv.unbind(dim=1)
|
||||
|
||||
k_cache = torch.zeros(cache_size,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype=dtype)
|
||||
v_cache = torch.zeros(cache_size,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype=dtype)
|
||||
k = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
|
||||
v = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
|
||||
values = torch.arange(0, cache_size, dtype=torch.long)
|
||||
values = values[torch.randperm(cache_size)]
|
||||
block_table = values[:batch_size * max_block_per_request].view(
|
||||
batch_size, max_block_per_request)
|
||||
b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long)
|
||||
b_start_loc = torch.cumsum(torch.tensor([0] + query_lens[:-1],
|
||||
dtype=torch.long),
|
||||
dim=0)
|
||||
# copy kv to cache
|
||||
b_seq_start_loc = torch.cumsum(torch.tensor([0] + seq_lens[:-1],
|
||||
dtype=torch.long),
|
||||
dim=0)
|
||||
for i in range(batch_size):
|
||||
for j in range(query_lens[i]):
|
||||
k[b_start_loc[i] + j].copy_(key[b_seq_start_loc[i] + b_ctx_len[i] +
|
||||
j])
|
||||
v[b_start_loc[i] + j].copy_(value[b_seq_start_loc[i] +
|
||||
b_ctx_len[i] + j])
|
||||
cur_ctx = 0
|
||||
block_id = 0
|
||||
while cur_ctx < b_ctx_len[i]:
|
||||
start_loc = b_seq_start_loc[i] + cur_ctx
|
||||
if cur_ctx + block_size > b_ctx_len[i]:
|
||||
end_loc = b_seq_start_loc[i] + b_ctx_len[i]
|
||||
else:
|
||||
end_loc = start_loc + block_size
|
||||
start_slot = block_table[i, block_id] * block_size
|
||||
end_slot = start_slot + end_loc - start_loc
|
||||
k_cache.view(-1, num_kv_heads,
|
||||
head_size)[start_slot:end_slot].copy_(
|
||||
key[start_loc:end_loc])
|
||||
v_cache.view(-1, num_kv_heads,
|
||||
head_size)[start_slot:end_slot].copy_(
|
||||
value[start_loc:end_loc])
|
||||
cur_ctx += block_size
|
||||
block_id += 1
|
||||
kv_cache = torch.stack([k_cache, v_cache])
|
||||
|
||||
return (
|
||||
query,
|
||||
k,
|
||||
v,
|
||||
kv_cache,
|
||||
block_table,
|
||||
key,
|
||||
value,
|
||||
query_lens,
|
||||
seq_lens,
|
||||
)
|
||||
|
||||
|
||||
def get_active_block_tables(block_tables, query_lens, seq_lens, block_size,
|
||||
num_blocks):
|
||||
context_lens = seq_lens - query_lens
|
||||
blocks_per_seq = (context_lens + block_size - 1) // block_size
|
||||
num_seqs = len(seq_lens)
|
||||
active_blocks: list[int] = []
|
||||
for seq_id in range(num_seqs):
|
||||
active_blocks = (
|
||||
active_blocks +
|
||||
block_tables[seq_id, :blocks_per_seq[seq_id]].tolist())
|
||||
return F.pad(
|
||||
torch.tensor(active_blocks, dtype=torch.int32),
|
||||
(0, num_blocks - len(active_blocks)),
|
||||
"constant",
|
||||
0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prefill_batch_size,decode_batch_size,block_size,large_tile_size,num_heads,num_queries_per_kv,head_size,mixed_precision",
|
||||
[
|
||||
# Test minimal configurations (small block size)
|
||||
(1, 199, 1, 512, 4, 2, 8, False
|
||||
), # minimal block size, small dimensions
|
||||
(1, 199, 1, 512, 4, 2, 8, True), # same with mixed precision
|
||||
|
||||
# Test common/medium configurations
|
||||
(4, 12, 32, 2048, 32, 8, 64, False), # common case, larger heads
|
||||
(4, 12, 32, 2048, 16, 4, 32,
|
||||
True), # medium size, mixed precision, grouped-query attention (GQA)
|
||||
|
||||
# Test large configurations
|
||||
(4, 12, 256, 8192, 8, 1, 128, False), # large blocks, large head size
|
||||
(4, 12, 256, 8192, 64, 8, 64, True), # large blocks, many heads
|
||||
|
||||
# Test asymmetric configurations
|
||||
(2, 24, 64, 4096, 12, 4, 96, False), # varied batch sizes
|
||||
(8, 8, 128, 2048, 24, 2, 48, True), # balanced batches
|
||||
|
||||
# Test edge cases
|
||||
(1, 128, 16, 1024, 4, 2, 16, False), # large decode batch
|
||||
(16, 4, 8, 1024, 4, 2, 128, True), # large prefill batch
|
||||
(4, 12, 32, 2048, 16, 1, 32, True), # multi-head attention (MHA)
|
||||
(4, 12, 32, 2048, 16, 16, 32, True), # multi-query attention (MQA)
|
||||
])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
prefill_batch_size: int,
|
||||
decode_batch_size: int,
|
||||
num_heads: int,
|
||||
num_queries_per_kv: int,
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
large_tile_size,
|
||||
mixed_precision: bool,
|
||||
) -> None:
|
||||
|
||||
import torch_xla.core.xla_model as xm
|
||||
|
||||
from vllm.attention.ops.nki_flash_attn import (flash_attn_varlen_nkifunc,
|
||||
reorder_context_mask)
|
||||
|
||||
assert large_tile_size % block_size == 0
|
||||
|
||||
device = xm.xla_device()
|
||||
|
||||
compiler_flags_str = " ".join([
|
||||
"-O1",
|
||||
"--retry_failed_compilation",
|
||||
])
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("NEURON_CC_FLAGS", compiler_flags_str)
|
||||
|
||||
torch.manual_seed(0)
|
||||
torch.set_printoptions(sci_mode=False)
|
||||
torch.set_default_device("cpu")
|
||||
dtype = torch.float32
|
||||
|
||||
min_ctx_len = 32
|
||||
max_ctx_len = 1024
|
||||
min_query_len = 16
|
||||
max_query_len = 512
|
||||
num_kv_heads = num_heads // num_queries_per_kv
|
||||
(
|
||||
query,
|
||||
k_active,
|
||||
v_active,
|
||||
kv_cache,
|
||||
block_table,
|
||||
key,
|
||||
value,
|
||||
query_lens,
|
||||
seq_lens,
|
||||
) = sample_inputs(
|
||||
prefill_batch_size=prefill_batch_size,
|
||||
decode_batch_size=decode_batch_size,
|
||||
min_query_len=min_query_len,
|
||||
max_query_len=max_query_len,
|
||||
min_ctx_len=min_ctx_len,
|
||||
max_ctx_len=max_ctx_len,
|
||||
block_size=block_size,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
output_ref = ref_context_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
query_lens,
|
||||
seq_lens,
|
||||
head_size,
|
||||
num_queries_per_kv,
|
||||
return_max_reduce=False,
|
||||
)
|
||||
|
||||
# build neuron program
|
||||
B_P_SIZE = 128
|
||||
assert (large_tile_size >= B_P_SIZE
|
||||
), f"Expect {large_tile_size=} to be larger than {B_P_SIZE=}"
|
||||
|
||||
def pad_to_multiple(a, b):
|
||||
return cdiv(a, b) * b
|
||||
|
||||
def pad_to_next_power_of_2(a):
|
||||
assert a > 0
|
||||
return 2**int(a - 1).bit_length()
|
||||
|
||||
# calculate input shapes
|
||||
max_num_queries = pad_to_next_power_of_2(sum(query_lens))
|
||||
context_lens = torch.tensor(seq_lens) - torch.tensor(query_lens)
|
||||
num_active_blocks = cdiv(context_lens, block_size).sum().item()
|
||||
num_active_blocks = pad_to_multiple(num_active_blocks,
|
||||
large_tile_size // block_size)
|
||||
context_kv_len = num_active_blocks * block_size
|
||||
assert (
|
||||
context_kv_len %
|
||||
large_tile_size == 0), f"invalid context_kv_len={context_kv_len}"
|
||||
|
||||
# pad QKV tensors
|
||||
pad_dims = (
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
max_num_queries - query.shape[0],
|
||||
)
|
||||
query = F.pad(query, pad_dims, "constant", 0)
|
||||
k = F.pad(k_active, pad_dims, "constant", 0)
|
||||
v = F.pad(v_active, pad_dims, "constant", 0)
|
||||
|
||||
# permute QKV tensors
|
||||
# query: (1, n_heads, d, seq_q)
|
||||
# key: (1, n_kv_heads, d, seq_k)
|
||||
# value: (1, n_kv_heads, seq_v, d)
|
||||
query = query.unsqueeze(0).permute(0, 2, 3, 1).contiguous()
|
||||
k = k.unsqueeze(0).permute(0, 2, 3, 1).contiguous()
|
||||
v = v.unsqueeze(0).permute(0, 2, 1, 3).contiguous()
|
||||
kv_cache = kv_cache.permute(0, 1, 3, 2, 4).contiguous()
|
||||
|
||||
# transform block table
|
||||
active_block_table = get_active_block_tables(
|
||||
block_table.cpu(),
|
||||
torch.tensor(query_lens).cpu(),
|
||||
torch.tensor(seq_lens).cpu(),
|
||||
block_size,
|
||||
num_active_blocks,
|
||||
)
|
||||
|
||||
# Build attention masks
|
||||
prior_mask, active_mask = (
|
||||
BlockDiagonalCausalFromBottomRightMask.from_seqlens(
|
||||
query_lens, seq_lens, block_size=block_size))
|
||||
prior_mask_padded = F.pad(
|
||||
prior_mask,
|
||||
(
|
||||
0,
|
||||
context_kv_len - prior_mask.shape[1],
|
||||
0,
|
||||
max_num_queries - prior_mask.shape[0],
|
||||
),
|
||||
"constant",
|
||||
0,
|
||||
).bool()
|
||||
active_mask_padded = F.pad(
|
||||
active_mask,
|
||||
(
|
||||
0,
|
||||
max_num_queries - active_mask.shape[1],
|
||||
0,
|
||||
max_num_queries - active_mask.shape[0],
|
||||
),
|
||||
"constant",
|
||||
0,
|
||||
).bool()
|
||||
attn_mask = torch.concat([prior_mask_padded, active_mask_padded],
|
||||
dim=1)
|
||||
|
||||
attn_mask = reorder_context_mask(attn_mask, large_tile_size,
|
||||
block_size)
|
||||
|
||||
input_args = (
|
||||
query.to(device=device),
|
||||
k.to(device=device),
|
||||
v.to(device=device),
|
||||
kv_cache.to(device=device),
|
||||
active_block_table.to(device=device),
|
||||
attn_mask.to(device=device),
|
||||
)
|
||||
input_kwargs = dict(
|
||||
n_kv_head=num_kv_heads,
|
||||
head_size=head_size,
|
||||
mixed_precision=mixed_precision,
|
||||
LARGE_TILE_SZ=large_tile_size,
|
||||
)
|
||||
|
||||
output_nki = flash_attn_varlen_nkifunc(*input_args, **input_kwargs)
|
||||
|
||||
num_actual_tokens = sum(query_lens)
|
||||
# - o: shape (bs, n_heads, seq_q, d) -> (bs, seq_q, n_heads, d)
|
||||
output_nki = output_nki.cpu().permute(0, 2, 1, 3)
|
||||
output_nki = output_nki[0, :num_actual_tokens, :, :]
|
||||
output_ref_padded = F.pad(
|
||||
output_ref,
|
||||
(0, 0, 0, 0, 0, 0, 0, max_num_queries - output_ref.shape[0]),
|
||||
"constant",
|
||||
0,
|
||||
)
|
||||
output_ref = output_ref_padded.transpose(
|
||||
0, 1)[0, :num_actual_tokens, :, :]
|
||||
|
||||
torch.testing.assert_close(output_nki, output_ref, atol=1e-2, rtol=0)
|
||||
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for miscellaneous utilities
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"max_position,is_neox_style,rotary_dim,head_size,seq_len,use_key", [
|
||||
(16, False, 32, 32, 1024, True),
|
||||
(16, False, 32, 128, 1024, True),
|
||||
(16, True, 32, 32, 1024, True),
|
||||
(16, True, 32, 128, 1024, True),
|
||||
(16, False, 32, 128, 1024, False),
|
||||
(16, True, 32, 128, 1024, False),
|
||||
])
|
||||
def test_rotary_embedding_opcheck(max_position, is_neox_style, rotary_dim,
|
||||
head_size, seq_len, use_key):
|
||||
import torch_xla.core.xla_model as xm
|
||||
|
||||
device = xm.xla_device()
|
||||
current_platform.seed_everything(0)
|
||||
torch.set_default_device("cpu")
|
||||
|
||||
batch_size = 1
|
||||
base = 10000
|
||||
num_heads = 8
|
||||
|
||||
rot = RotaryEmbedding(head_size, rotary_dim, max_position, base,
|
||||
is_neox_style, torch.float32)
|
||||
|
||||
positions = torch.randint(0,
|
||||
max_position, (batch_size, seq_len),
|
||||
device="cpu")
|
||||
query = torch.randn(batch_size,
|
||||
seq_len,
|
||||
num_heads * head_size,
|
||||
dtype=torch.float32,
|
||||
device="cpu")
|
||||
key = torch.randn_like(query) if use_key else None
|
||||
assert positions.is_cpu, \
|
||||
"reference input tensor is expected to be CPU tensor."
|
||||
ref_query, ref_key = rot.to(device="cpu").forward_native(
|
||||
positions, query, key)
|
||||
out_query, out_key = rot.to(device=device).forward_neuron(
|
||||
positions.to(device=device), query.to(device=device),
|
||||
key.to(device=device) if key is not None else None)
|
||||
if use_key:
|
||||
assert out_query.is_xla and out_key.is_xla, \
|
||||
"output tensor is expected to be XLA tensor"
|
||||
torch.testing.assert_close(out_key.cpu(),
|
||||
ref_key,
|
||||
atol=1e-2,
|
||||
rtol=1e-2)
|
||||
else:
|
||||
assert out_key is None, "expected returned key to be None"
|
||||
assert out_query.is_xla, \
|
||||
"output tensor is expected to be XLA tensor"
|
||||
torch.testing.assert_close(out_query.cpu(),
|
||||
ref_query,
|
||||
atol=1e-2,
|
||||
rtol=1e-2)
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
from typing import Callable
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch_xla.distributed.xla_multiprocessing as xmp
|
||||
from typing_extensions import ParamSpec
|
||||
|
||||
from vllm.distributed.communication_op import (
|
||||
tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce)
|
||||
from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
|
||||
init_distributed_environment)
|
||||
from vllm.utils import get_distributed_init_method, get_open_port
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
|
||||
|
||||
def reinitialize_neuron_runtime(f: Callable[_P, None]) -> Callable[_P, None]:
|
||||
"""Decorator to reinitialize the Neuron Runtime before executing a test.
|
||||
This is necessary for distributed tests which need to reallocate Neuron
|
||||
Cores to separate subprocesses.
|
||||
"""
|
||||
|
||||
@functools.wraps(f)
|
||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
|
||||
runtime = torch.classes.neuron.Runtime()
|
||||
runtime.initialize()
|
||||
runtime.unsafe_close()
|
||||
|
||||
f(*args, **kwargs)
|
||||
runtime.initialize()
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def all_gather_test_worker(index, tp_degree, distributed_init_method):
|
||||
init_distributed_environment(tp_degree,
|
||||
index,
|
||||
distributed_init_method,
|
||||
index,
|
||||
backend="xla")
|
||||
ensure_model_parallel_initialized(tp_degree, 1)
|
||||
|
||||
num_dimensions = 3
|
||||
tensor_size = list(range(2, num_dimensions + 2))
|
||||
total_size = 1
|
||||
for s in tensor_size:
|
||||
total_size *= s
|
||||
|
||||
all_gather_dimension = -1
|
||||
all_tensors = [
|
||||
torch.arange(total_size, dtype=torch.float32,
|
||||
device="xla").reshape(tensor_size) * (r + 1)
|
||||
for r in range(tp_degree)
|
||||
]
|
||||
expected = torch.cat(all_tensors, dim=all_gather_dimension)
|
||||
t = all_tensors[index % tp_degree]
|
||||
t = tensor_model_parallel_all_gather(t, all_gather_dimension)
|
||||
torch.testing.assert_close(t, expected)
|
||||
|
||||
|
||||
def all_reduce_test_worker(index, tp_degree, distributed_init_method):
|
||||
init_distributed_environment(tp_degree,
|
||||
index,
|
||||
distributed_init_method,
|
||||
index,
|
||||
backend="xla")
|
||||
ensure_model_parallel_initialized(tp_degree, 1)
|
||||
|
||||
num_elements = 8
|
||||
all_tensors = [
|
||||
torch.arange(num_elements, dtype=torch.float32, device="xla") * (r + 1)
|
||||
for r in range(tp_degree)
|
||||
]
|
||||
expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
|
||||
t = all_tensors[index % tp_degree]
|
||||
t = tensor_model_parallel_all_reduce(t)
|
||||
torch.testing.assert_close(t, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("test_target",
|
||||
[all_reduce_test_worker, all_gather_test_worker])
|
||||
@reinitialize_neuron_runtime
|
||||
def test_neuron_multi_process_tensor_parallel(monkeypatch, tp_size,
|
||||
test_target):
|
||||
|
||||
with patch('torch_xla._XLAC._xla_runtime_is_initialized',
|
||||
return_value=False):
|
||||
distributed_init_method = get_distributed_init_method(
|
||||
"127.0.0.1", get_open_port())
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V1", "1")
|
||||
monkeypatch.setenv("NEURONCORE_NUM_DEVICES", str(tp_size))
|
||||
monkeypatch.setenv("NEURON_PJRT_PROCESSES_NUM_DEVICES",
|
||||
','.join(['1' for _ in range(tp_size)]))
|
||||
|
||||
xmp.spawn(test_target, args=(tp_size, distributed_init_method))
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
import torch
|
||||
from huggingface_hub import snapshot_download
|
||||
from safetensors import safe_open
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
|
||||
def patch_eagle_draft_with_lm_head(target_model_id: str,
|
||||
draft_model_id: str) -> str:
|
||||
# In NxDI, draft model checkpoint must include lm_head weights from target
|
||||
# model. For more details see https://awsdocs-neuron.readthedocs-hosted.com
|
||||
# /en/latest/libraries/nxd-inference/developer_guides/feature-guide.html
|
||||
# #eagle-checkpoint-compatibility
|
||||
final_draft_dir = "/tmp/patched_eagle_draft"
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
target_dir = snapshot_download(repo_id=target_model_id,
|
||||
local_dir=os.path.join(
|
||||
tmp_dir, "target"))
|
||||
draft_dir = snapshot_download(repo_id=draft_model_id,
|
||||
local_dir=os.path.join(tmp_dir, "draft"))
|
||||
|
||||
lm_head_key = "lm_head.weight"
|
||||
index_path = os.path.join(target_dir, "model.safetensors.index.json")
|
||||
with open(index_path) as f:
|
||||
index = json.load(f)
|
||||
shard_name = index["weight_map"][lm_head_key]
|
||||
target_safetensor_path = os.path.join(target_dir, shard_name)
|
||||
|
||||
with safe_open(target_safetensor_path, framework="pt") as f:
|
||||
target_lm_head = f.get_tensor(lm_head_key)
|
||||
|
||||
draft_path = os.path.join(draft_dir, "pytorch_model.bin")
|
||||
draft_state_dict = torch.load(draft_path, map_location="cpu")
|
||||
draft_state_dict[lm_head_key] = target_lm_head.to(torch.float16)
|
||||
torch.save(draft_state_dict, draft_path)
|
||||
|
||||
shutil.copytree(draft_dir, final_draft_dir, dirs_exist_ok=True)
|
||||
|
||||
return final_draft_dir
|
||||
|
||||
|
||||
def test_eagle():
|
||||
patched_draft_path = patch_eagle_draft_with_lm_head(
|
||||
target_model_id="meta-llama/Llama-2-7b-hf",
|
||||
draft_model_id="yuhuili/EAGLE-llama2-chat-7B")
|
||||
llm = LLM(
|
||||
model="meta-llama/Llama-2-7b-hf",
|
||||
speculative_config={
|
||||
"model": patched_draft_path,
|
||||
"num_speculative_tokens": 5,
|
||||
"max_model_len": 128
|
||||
},
|
||||
max_num_seqs=1,
|
||||
max_model_len=128,
|
||||
tensor_parallel_size=2,
|
||||
override_neuron_config={
|
||||
"enable_eagle_speculation": True,
|
||||
"enable_fused_speculation": True,
|
||||
"fused_qkv": True
|
||||
},
|
||||
)
|
||||
prompts = [
|
||||
"The president of the United States is",
|
||||
]
|
||||
outputs = llm.generate(prompts, SamplingParams(top_k=1))
|
||||
expected_output = " the head of state and head of government of " \
|
||||
"the United States. The president direct"
|
||||
|
||||
for output in outputs:
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
|
||||
assert (expected_output == generated_text)
|
||||
|
||||
print("Neuron Eagle speculation test passed.")
|
||||
@@ -1,64 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
|
||||
def test_mistral():
|
||||
llm = LLM(model="mistralai/Mistral-7B-v0.1",
|
||||
tensor_parallel_size=2,
|
||||
max_num_seqs=4,
|
||||
max_model_len=128,
|
||||
override_neuron_config={
|
||||
"sequence_parallel_enabled": False,
|
||||
"skip_warmup": True
|
||||
})
|
||||
|
||||
# Send more prompts than the compiled batch size (4) and request
|
||||
# varying generation lengths to test accuracy related to Neuron
|
||||
# specific sequence id sorting.
|
||||
prompts = [
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"What is Annapurna labs?",
|
||||
"I believe the meaning of life is",
|
||||
"Tell me a story about a brave knight",
|
||||
"Hello, my name is Llama",
|
||||
]
|
||||
|
||||
sampling_params = [
|
||||
SamplingParams(top_k=1, max_tokens=10),
|
||||
SamplingParams(top_k=1, max_tokens=20),
|
||||
SamplingParams(top_k=1, max_tokens=30),
|
||||
SamplingParams(top_k=1, max_tokens=40),
|
||||
SamplingParams(top_k=1, max_tokens=50),
|
||||
SamplingParams(top_k=1, max_tokens=60)
|
||||
]
|
||||
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
expected_outputs = [
|
||||
" the most powerful person in the world. He is",
|
||||
" a city of many faces. It is a city of history, culture, art, "
|
||||
"fashion, and",
|
||||
"\n\nAnnapurna Labs is a semiconductor company that was founded "
|
||||
"in 2013 by Amazon. The company is",
|
||||
" to be happy.\n\nI believe that happiness is a choice.\n\nI "
|
||||
"believe that happiness is a state of mind.\n\nI believe that "
|
||||
"happiness is a journey.\n\nI believe",
|
||||
" who rescued a princess from a dragon.\n\nTell me a story about"
|
||||
" a princess who rescued herself from a dragon.\n\nTell me a "
|
||||
"story about a princess who rescued herself from a dragon and "
|
||||
"then rescued a knight from",
|
||||
" and I am a 10 year old male. I am a very friendly and "
|
||||
"affectionate boy who loves to be around people. I am a very "
|
||||
"active boy who loves to play and run around. I am a very smart "
|
||||
"boy who loves to learn new things. I am a very loyal boy"
|
||||
]
|
||||
|
||||
for expected_output, output in zip(expected_outputs, outputs):
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
|
||||
assert (expected_output == generated_text)
|
||||
|
||||
print("Neuron Mistral test passed.")
|
||||
@@ -1,97 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
|
||||
def test_llama_single_lora():
|
||||
sql_lora_files = snapshot_download(
|
||||
repo_id="yard1/llama-2-7b-sql-lora-test")
|
||||
llm = LLM(model="meta-llama/Llama-2-7b-hf",
|
||||
tensor_parallel_size=2,
|
||||
max_num_seqs=4,
|
||||
max_model_len=512,
|
||||
override_neuron_config={
|
||||
"sequence_parallel_enabled": False,
|
||||
"skip_warmup": True,
|
||||
"lora_modules": [{
|
||||
"name": "lora_id_1",
|
||||
"path": sql_lora_files
|
||||
}]
|
||||
},
|
||||
enable_lora=True,
|
||||
max_loras=1,
|
||||
max_lora_rank=256,
|
||||
device="neuron")
|
||||
"""For multi-lora requests using NxDI as the backend, only the lora_name
|
||||
needs to be specified. The lora_id and lora_path are supplied at the LLM
|
||||
class/server initialization, after which the paths are handled by NxDI"""
|
||||
lora_req_1 = LoRARequest("lora_id_1", 0, " ")
|
||||
prompts = [
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
]
|
||||
outputs = llm.generate(prompts,
|
||||
SamplingParams(top_k=1),
|
||||
lora_request=[lora_req_1, lora_req_1])
|
||||
|
||||
expected_outputs = [
|
||||
" the head of state and head of government of the United States. "
|
||||
"The president direct",
|
||||
" a city of contrasts. The city is home to the Eiffel Tower"
|
||||
]
|
||||
|
||||
for expected_output, output in zip(expected_outputs, outputs):
|
||||
generated_text = output.outputs[0].text
|
||||
assert (expected_output == generated_text)
|
||||
|
||||
|
||||
def test_llama_multiple_lora():
|
||||
sql_lora_files = snapshot_download(
|
||||
repo_id="yard1/llama-2-7b-sql-lora-test")
|
||||
llm = LLM(model="meta-llama/Llama-2-7b-hf",
|
||||
tensor_parallel_size=2,
|
||||
max_num_seqs=4,
|
||||
max_model_len=512,
|
||||
override_neuron_config={
|
||||
"sequence_parallel_enabled":
|
||||
False,
|
||||
"skip_warmup":
|
||||
True,
|
||||
"lora_modules": [{
|
||||
"name": "lora_id_1",
|
||||
"path": sql_lora_files
|
||||
}, {
|
||||
"name": "lora_id_2",
|
||||
"path": sql_lora_files
|
||||
}]
|
||||
},
|
||||
enable_lora=True,
|
||||
max_loras=2,
|
||||
max_lora_rank=256,
|
||||
device="neuron")
|
||||
"""For multi-lora requests using NxDI as the backend, only the lora_name
|
||||
needs to be specified. The lora_id and lora_path are supplied at the LLM
|
||||
class/server initialization, after which the paths are handled by NxDI"""
|
||||
lora_req_1 = LoRARequest("lora_id_1", 0, " ")
|
||||
lora_req_2 = LoRARequest("lora_id_2", 1, " ")
|
||||
prompts = [
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
]
|
||||
outputs = llm.generate(prompts,
|
||||
SamplingParams(top_k=1),
|
||||
lora_request=[lora_req_1, lora_req_2])
|
||||
|
||||
expected_outputs = [
|
||||
" the head of state and head of government of the United States. "
|
||||
"The president direct",
|
||||
" a city of contrasts. The city is home to the Eiffel Tower"
|
||||
]
|
||||
|
||||
for expected_output, output in zip(expected_outputs, outputs):
|
||||
generated_text = output.outputs[0].text
|
||||
assert (expected_output == generated_text)
|
||||
Reference in New Issue
Block a user