Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+51
-39
@@ -4,6 +4,7 @@
|
||||
|
||||
Run `pytest tests/v1/tpu/test_basic.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -32,8 +33,9 @@ MAX_NUM_REQS = [16, 1024]
|
||||
# TENSOR_PARALLEL_SIZES = [1, 4]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="This is a basic test for TPU only")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_tpu(), reason="This is a basic test for TPU only"
|
||||
)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("max_tokens", [5])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", TENSOR_PARALLEL_SIZES)
|
||||
@@ -46,32 +48,36 @@ def test_basic(
|
||||
tensor_parallel_size: int,
|
||||
max_num_seqs: int,
|
||||
) -> None:
|
||||
prompt = "The next numbers of the sequence " + ", ".join(
|
||||
str(i) for i in range(1024)) + " are:"
|
||||
prompt = (
|
||||
"The next numbers of the sequence "
|
||||
+ ", ".join(str(i) for i in range(1024))
|
||||
+ " are:"
|
||||
)
|
||||
example_prompts = [prompt]
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_USE_V1", "1")
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
# Note: max_num_batched_tokens == 1024 is needed here to
|
||||
# actually test chunked prompt
|
||||
max_num_batched_tokens=1024,
|
||||
max_model_len=8192,
|
||||
gpu_memory_utilization=0.7,
|
||||
max_num_seqs=max_num_seqs,
|
||||
tensor_parallel_size=tensor_parallel_size) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy(example_prompts,
|
||||
max_tokens)
|
||||
model,
|
||||
# Note: max_num_batched_tokens == 1024 is needed here to
|
||||
# actually test chunked prompt
|
||||
max_num_batched_tokens=1024,
|
||||
max_model_len=8192,
|
||||
gpu_memory_utilization=0.7,
|
||||
max_num_seqs=max_num_seqs,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
|
||||
output = vllm_outputs[0][1]
|
||||
|
||||
assert "1024" in output or "0, 1" in output
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Temporarily disabled due to timeout")
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="This is a basic test for TPU only")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_tpu(), reason="This is a basic test for TPU only"
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [8])
|
||||
@pytest.mark.parametrize("max_num_seqs", [16])
|
||||
def test_phi3(
|
||||
@@ -96,9 +102,9 @@ def test_phi3(
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_USE_V1", "1")
|
||||
|
||||
with vllm_runner(model,
|
||||
max_num_batched_tokens=256,
|
||||
max_num_seqs=max_num_seqs) as vllm_model:
|
||||
with vllm_runner(
|
||||
model, max_num_batched_tokens=256, max_num_seqs=max_num_seqs
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy(prompts, max_tokens)
|
||||
# vllm_outputs is a list of tuples whose first element is the token id
|
||||
# and the second element is the output (including the prompt).
|
||||
@@ -110,10 +116,11 @@ def test_phi3(
|
||||
TP_SIZE_8 = 8
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="This is a test for TPU only")
|
||||
@pytest.mark.skipif(tpu.num_available_chips() < TP_SIZE_8,
|
||||
reason=f"This test requires {TP_SIZE_8} TPU chips.")
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(), reason="This is a test for TPU only")
|
||||
@pytest.mark.skipif(
|
||||
tpu.num_available_chips() < TP_SIZE_8,
|
||||
reason=f"This test requires {TP_SIZE_8} TPU chips.",
|
||||
)
|
||||
def test_gemma3_27b_with_text_input_and_tp(
|
||||
vllm_runner: type[VllmRunner],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -137,10 +144,11 @@ def test_gemma3_27b_with_text_input_and_tp(
|
||||
m.setenv("VLLM_USE_V1", "1")
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_num_batched_tokens=256,
|
||||
max_num_seqs=max_num_seqs,
|
||||
tensor_parallel_size=tensor_parallel_size) as vllm_model:
|
||||
model,
|
||||
max_num_batched_tokens=256,
|
||||
max_num_seqs=max_num_seqs,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy(prompts, max_tokens)
|
||||
# vllm_outputs is a list of tuples whose first element is the token id
|
||||
# and the second element is the output (including the prompt).
|
||||
@@ -149,8 +157,9 @@ def test_gemma3_27b_with_text_input_and_tp(
|
||||
assert answer in generated_text
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="This is a basic test for TPU only")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_tpu(), reason="This is a basic test for TPU only"
|
||||
)
|
||||
def test_w8a8_quantization(
|
||||
vllm_runner: type[VllmRunner],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -160,22 +169,25 @@ def test_w8a8_quantization(
|
||||
tensor_parallel_size = 1
|
||||
max_num_seqs = 4
|
||||
|
||||
prompt = "The next numbers of the sequence " + ", ".join(
|
||||
str(i) for i in range(1024)) + " are:"
|
||||
prompt = (
|
||||
"The next numbers of the sequence "
|
||||
+ ", ".join(str(i) for i in range(1024))
|
||||
+ " are:"
|
||||
)
|
||||
example_prompts = [prompt]
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_USE_V1", "1")
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_num_batched_tokens=64,
|
||||
max_model_len=4096,
|
||||
gpu_memory_utilization=0.7,
|
||||
max_num_seqs=max_num_seqs,
|
||||
tensor_parallel_size=tensor_parallel_size) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy(example_prompts,
|
||||
max_tokens)
|
||||
model,
|
||||
max_num_batched_tokens=64,
|
||||
max_model_len=4096,
|
||||
gpu_memory_utilization=0.7,
|
||||
max_num_seqs=max_num_seqs,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
|
||||
output = vllm_outputs[0][1]
|
||||
|
||||
assert "1024" in output or "0, 1" in output
|
||||
|
||||
@@ -10,61 +10,69 @@ import vllm.v1.attention.backends.pallas # noqa: F401
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="This is a test for TPU only")
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(), reason="This is a test for TPU only")
|
||||
@pytest.mark.parametrize("page_size", [32, 33])
|
||||
@pytest.mark.parametrize("combined_kv_head_num", [2, 16])
|
||||
@pytest.mark.parametrize("head_dim", [128, 256])
|
||||
@pytest.mark.parametrize("num_slices_per_block", [4, 8])
|
||||
def test_kv_cache_update_kernel(page_size: int, combined_kv_head_num: int,
|
||||
head_dim: int, num_slices_per_block: int):
|
||||
def test_kv_cache_update_kernel(
|
||||
page_size: int, combined_kv_head_num: int, head_dim: int, num_slices_per_block: int
|
||||
):
|
||||
page_num = 1000
|
||||
padded_num_tokens = 128
|
||||
kv_cache_cpu = torch.zeros(
|
||||
(page_num * page_size, combined_kv_head_num, head_dim),
|
||||
dtype=torch.bfloat16,
|
||||
device="cpu")
|
||||
device="cpu",
|
||||
)
|
||||
kv_cache_xla = kv_cache_cpu.to(torch_xla.device())
|
||||
new_kv_cpu = torch.randn(
|
||||
(padded_num_tokens, combined_kv_head_num, head_dim),
|
||||
dtype=torch.bfloat16,
|
||||
device="cpu")
|
||||
device="cpu",
|
||||
)
|
||||
new_kv_xla = new_kv_cpu.to(torch_xla.device())
|
||||
slice_lens = np.array([7, page_size, page_size, 1, 1, 1, 9],
|
||||
dtype=np.int32)
|
||||
slice_lens = np.array([7, page_size, page_size, 1, 1, 1, 9], dtype=np.int32)
|
||||
num_kv_update_slices = len(slice_lens)
|
||||
kv_cache_start_indices = np.array([
|
||||
page_size * 2 - 7, page_size * 2, page_size * 3, page_size * 4 + 6,
|
||||
page_size * 5 + 7, page_size * 6 + 8, page_size * 15 + 3
|
||||
],
|
||||
dtype=np.int32)
|
||||
kv_cache_start_indices = np.array(
|
||||
[
|
||||
page_size * 2 - 7,
|
||||
page_size * 2,
|
||||
page_size * 3,
|
||||
page_size * 4 + 6,
|
||||
page_size * 5 + 7,
|
||||
page_size * 6 + 8,
|
||||
page_size * 15 + 3,
|
||||
],
|
||||
dtype=np.int32,
|
||||
)
|
||||
new_kv_cache_indices = np.concatenate(
|
||||
[np.array([0], dtype=np.int32),
|
||||
np.cumsum(slice_lens[:-1])])
|
||||
[np.array([0], dtype=np.int32), np.cumsum(slice_lens[:-1])]
|
||||
)
|
||||
slot_mapping = np.stack(
|
||||
[kv_cache_start_indices, new_kv_cache_indices, slice_lens], axis=1)
|
||||
[kv_cache_start_indices, new_kv_cache_indices, slice_lens], axis=1
|
||||
)
|
||||
slot_mapping = np.transpose(slot_mapping)
|
||||
slot_mapping_cpu = torch.tensor(slot_mapping,
|
||||
device="cpu",
|
||||
dtype=torch.int32)
|
||||
slot_mapping_cpu = torch.tensor(slot_mapping, device="cpu", dtype=torch.int32)
|
||||
slot_mapping_xla = slot_mapping_cpu.to(torch_xla.device())
|
||||
num_kv_update_slices_xla = torch.tensor([num_kv_update_slices],
|
||||
device=torch_xla.device(),
|
||||
dtype=torch.int32)
|
||||
num_kv_update_slices_xla = torch.tensor(
|
||||
[num_kv_update_slices], device=torch_xla.device(), dtype=torch.int32
|
||||
)
|
||||
torch_xla.sync()
|
||||
|
||||
torch.ops.xla.dynamo_set_buffer_donor_(kv_cache_xla, True)
|
||||
new_kv_cache_xla = torch.ops.xla.kv_cache_update_op(
|
||||
new_kv_xla, slot_mapping_xla, kv_cache_xla, num_kv_update_slices_xla,
|
||||
page_size, num_slices_per_block)
|
||||
new_kv_xla,
|
||||
slot_mapping_xla,
|
||||
kv_cache_xla,
|
||||
num_kv_update_slices_xla,
|
||||
page_size,
|
||||
num_slices_per_block,
|
||||
)
|
||||
kv_cache_xla.copy_(new_kv_cache_xla)
|
||||
torch_xla.sync()
|
||||
|
||||
for ni, ci, sl in zip(new_kv_cache_indices, kv_cache_start_indices,
|
||||
slice_lens):
|
||||
kv_cache_cpu[ci:ci + sl, :, :] = new_kv_cpu[ni:ni + sl, :, :]
|
||||
for ni, ci, sl in zip(new_kv_cache_indices, kv_cache_start_indices, slice_lens):
|
||||
kv_cache_cpu[ci : ci + sl, :, :] = new_kv_cpu[ni : ni + sl, :, :]
|
||||
|
||||
assert torch.allclose(kv_cache_xla.cpu(),
|
||||
kv_cache_cpu,
|
||||
atol=1e-4,
|
||||
rtol=1e-4)
|
||||
assert torch.allclose(kv_cache_xla.cpu(), kv_cache_cpu, atol=1e-4, rtol=1e-4)
|
||||
|
||||
@@ -19,8 +19,7 @@ from vllm.platforms import current_platform
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_cache():
|
||||
"""Clear lru cache to ensure each test case runs without caching.
|
||||
"""
|
||||
"""Clear lru cache to ensure each test case runs without caching."""
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
|
||||
@@ -49,8 +48,7 @@ NUM_KV_HEADS = [1]
|
||||
HEAD_SIZES = [64, 80]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="This test needs a TPU")
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(), reason="This test needs a TPU")
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
|
||||
@pytest.mark.parametrize("seq_len", SEQ_LENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@@ -68,19 +66,12 @@ def test_mha_attn_forward(
|
||||
current_platform.seed_everything(0)
|
||||
# These are expected to be f32
|
||||
q = torch.randn(batch_size, seq_len, num_heads * head_size, device=device)
|
||||
k = torch.randn(batch_size,
|
||||
seq_len,
|
||||
num_kv_heads * head_size,
|
||||
device=device)
|
||||
v = torch.randn(batch_size,
|
||||
seq_len,
|
||||
num_kv_heads * head_size,
|
||||
device=device)
|
||||
k = torch.randn(batch_size, seq_len, num_kv_heads * head_size, device=device)
|
||||
v = torch.randn(batch_size, seq_len, num_kv_heads * head_size, device=device)
|
||||
scale = 1.0 / head_size**0.5
|
||||
attn = MultiHeadAttention(num_heads,
|
||||
head_size,
|
||||
scale=scale,
|
||||
num_kv_heads=num_kv_heads)
|
||||
attn = MultiHeadAttention(
|
||||
num_heads, head_size, scale=scale, num_kv_heads=num_kv_heads
|
||||
)
|
||||
output = attn(q, k, v)
|
||||
|
||||
assert num_heads % num_kv_heads == 0
|
||||
|
||||
@@ -14,38 +14,32 @@ from ...utils import RemoteOpenAIServer
|
||||
@pytest.fixture(scope="session")
|
||||
def base64_encoded_image(local_asset_server) -> dict[str, str]:
|
||||
return {
|
||||
image_asset:
|
||||
encode_image_base64(local_asset_server.get_image_asset(image_asset))
|
||||
image_asset: encode_image_base64(
|
||||
local_asset_server.get_image_asset(image_asset)
|
||||
)
|
||||
for image_asset in TEST_IMAGE_ASSETS
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="This test needs a TPU")
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(), reason="This test needs a TPU")
|
||||
@pytest.mark.parametrize("model_name", ["llava-hf/llava-1.5-7b-hf"])
|
||||
async def test_basic_vision(model_name: str, base64_encoded_image: dict[str,
|
||||
str]):
|
||||
|
||||
async def test_basic_vision(model_name: str, base64_encoded_image: dict[str, str]):
|
||||
pytest.skip("Skip this test until it's fixed.")
|
||||
|
||||
def whats_in_this_image_msg(b64):
|
||||
return [{
|
||||
"role":
|
||||
"user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What's in this image?"
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/jpeg;base64,{b64}"
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{b64}"},
|
||||
},
|
||||
},
|
||||
],
|
||||
}]
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
server_args = [
|
||||
"--max-model-len",
|
||||
@@ -62,19 +56,20 @@ async def test_basic_vision(model_name: str, base64_encoded_image: dict[str,
|
||||
]
|
||||
|
||||
# Server will pre-compile on first startup (takes a long time).
|
||||
with RemoteOpenAIServer(model_name, server_args,
|
||||
max_wait_seconds=600) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
model_name, server_args, max_wait_seconds=600
|
||||
) as remote_server:
|
||||
client: openai.AsyncOpenAI = remote_server.get_async_client()
|
||||
|
||||
# Other requests now should be much faster
|
||||
for image_url in TEST_IMAGE_ASSETS:
|
||||
image_base64 = base64_encoded_image[image_url]
|
||||
chat_completion_from_base64 = await client.chat.completions\
|
||||
.create(
|
||||
chat_completion_from_base64 = await client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=whats_in_this_image_msg(image_base64),
|
||||
max_completion_tokens=24,
|
||||
temperature=0.0)
|
||||
temperature=0.0,
|
||||
)
|
||||
result = chat_completion_from_base64
|
||||
assert result
|
||||
choice = result.choices[0]
|
||||
|
||||
@@ -5,8 +5,7 @@ from unittest.mock import ANY, patch
|
||||
import torch
|
||||
|
||||
from vllm.attention.backends.abstract import AttentionType
|
||||
from vllm.v1.attention.backends.pallas import (PallasAttentionBackendImpl,
|
||||
PallasMetadata)
|
||||
from vllm.v1.attention.backends.pallas import PallasAttentionBackendImpl, PallasMetadata
|
||||
|
||||
|
||||
def test_ragged_paged_attention():
|
||||
@@ -53,14 +52,14 @@ def test_ragged_paged_attention():
|
||||
max_num_reqs = 8
|
||||
max_num_blocks_per_req = 8
|
||||
num_kv_update_slices = torch.tensor([num_tokens], dtype=torch.int32)
|
||||
block_tables = torch.zeros((max_num_reqs, max_num_blocks_per_req),
|
||||
dtype=torch.int32)
|
||||
context_lens = torch.ones((max_num_reqs, ), dtype=torch.int32)
|
||||
block_tables = torch.zeros(
|
||||
(max_num_reqs, max_num_blocks_per_req), dtype=torch.int32
|
||||
)
|
||||
context_lens = torch.ones((max_num_reqs,), dtype=torch.int32)
|
||||
query_lens = [1] * max_num_reqs
|
||||
query_start_loc = torch.cumsum(torch.tensor([0] + query_lens,
|
||||
dtype=torch.int32),
|
||||
dim=0,
|
||||
dtype=torch.int32)
|
||||
query_start_loc = torch.cumsum(
|
||||
torch.tensor([0] + query_lens, dtype=torch.int32), dim=0, dtype=torch.int32
|
||||
)
|
||||
num_seqs = torch.tensor([max_num_reqs], dtype=torch.int32)
|
||||
attn_metadata = PallasMetadata(
|
||||
slot_mapping=slot_mapping,
|
||||
@@ -72,8 +71,7 @@ def test_ragged_paged_attention():
|
||||
num_slices_per_kv_cache_update_block=8,
|
||||
)
|
||||
|
||||
with patch("torch.ops.xla.ragged_paged_attention"
|
||||
) as mock_ragged_paged_attention:
|
||||
with patch("torch.ops.xla.ragged_paged_attention") as mock_ragged_paged_attention:
|
||||
attn_impl.forward(
|
||||
layer=layer,
|
||||
query=query,
|
||||
|
||||
+37
-29
@@ -4,6 +4,7 @@
|
||||
|
||||
Run `pytest tests/v1/tpu/test_perf.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
@@ -37,7 +38,6 @@ TEST_PARAMS = [
|
||||
# open(/dev/vfio/0): Device or resource busy: Device or resource busy;
|
||||
# Couldn't open iommu group /dev/vfio/0
|
||||
# => Investigate
|
||||
|
||||
# TestParams(
|
||||
# model="Qwen/Qwen2.5-1.5B-Instruct",
|
||||
# num_prompts=1,
|
||||
@@ -59,16 +59,14 @@ TEST_PARAMS = [
|
||||
num_prompts=64,
|
||||
prefix_len=500,
|
||||
decode_len=50,
|
||||
|
||||
# commit id: ccb246776d93ef105904a8ec015b3587240a1183
|
||||
# tpu: v5lite (old vllm CI/CD)
|
||||
# expected_avg_time=1.4,
|
||||
# err_tol=0.30,
|
||||
|
||||
# (This is the active CI/CD instance)
|
||||
# commit id: ccb246776d93ef105904a8ec015b3587240a1183
|
||||
# tpu: v6e (current vllm CI/CD)
|
||||
expected_avg_time=1.7, # measured with VLLM_XLA_CACHE_PATH=
|
||||
expected_avg_time=1.7, # measured with VLLM_XLA_CACHE_PATH=
|
||||
err_tol=0.20,
|
||||
),
|
||||
]
|
||||
@@ -81,44 +79,50 @@ MAX_NUM_SEQS = 32
|
||||
GPU_UTIL = 0.9
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="This is a basic performance test for TPU only")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_tpu(),
|
||||
reason="This is a basic performance test for TPU only",
|
||||
)
|
||||
@pytest.mark.parametrize("params", TEST_PARAMS)
|
||||
def test_perf(
|
||||
vllm_runner: type[VllmRunner],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
params: TestParams,
|
||||
) -> None:
|
||||
tokenizer = get_tokenizer(params.model,
|
||||
tokenizer_mode="auto",
|
||||
trust_remote_code=True)
|
||||
tokenizer = get_tokenizer(
|
||||
params.model, tokenizer_mode="auto", trust_remote_code=True
|
||||
)
|
||||
|
||||
prompts = []
|
||||
for i in range(params.num_prompts):
|
||||
prefix_token_ids = np.random.randint(0,
|
||||
tokenizer.vocab_size,
|
||||
size=params.prefix_len).tolist()
|
||||
prefix_token_ids = np.random.randint(
|
||||
0, tokenizer.vocab_size, size=params.prefix_len
|
||||
).tolist()
|
||||
prompt = tokenizer.decode(prefix_token_ids)
|
||||
prompts.append(prompt)
|
||||
|
||||
print(
|
||||
"-- Running: num_prompts = {} prefix_len = {} decode_len = {}".format(
|
||||
len(prompts), params.prefix_len, params.decode_len))
|
||||
len(prompts), params.prefix_len, params.decode_len
|
||||
)
|
||||
)
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_USE_V1", "1")
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=params.decode_len,
|
||||
temperature=1.0,
|
||||
min_p=0.0)
|
||||
sampling_params = SamplingParams(
|
||||
max_tokens=params.decode_len, temperature=1.0, min_p=0.0
|
||||
)
|
||||
|
||||
with vllm_runner(params.model,
|
||||
max_num_batched_tokens=MAX_MODEL_LEN,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
gpu_memory_utilization=GPU_UTIL,
|
||||
enforce_eager=False,
|
||||
tensor_parallel_size=1) as vllm_model:
|
||||
with vllm_runner(
|
||||
params.model,
|
||||
max_num_batched_tokens=MAX_MODEL_LEN,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
gpu_memory_utilization=GPU_UTIL,
|
||||
enforce_eager=False,
|
||||
tensor_parallel_size=1,
|
||||
) as vllm_model:
|
||||
print(" -- Warmup / Compile")
|
||||
for i in range(NUM_WARMUPS):
|
||||
_ = vllm_model.generate(prompts, sampling_params)
|
||||
@@ -133,14 +137,18 @@ def test_perf(
|
||||
avg_time = sum(times) / len(times)
|
||||
|
||||
print(" -- avg_time = {}".format(avg_time))
|
||||
print(" -- expected_avg_time = {} with err_tol = {}".format(
|
||||
params.expected_avg_time, params.err_tol))
|
||||
print(
|
||||
" -- expected_avg_time = {} with err_tol = {}".format(
|
||||
params.expected_avg_time, params.err_tol
|
||||
)
|
||||
)
|
||||
diff = avg_time - params.expected_avg_time
|
||||
ok = diff < params.err_tol
|
||||
if diff < -params.err_tol:
|
||||
print(" !! WARNING !! Performance has improved by {}, "
|
||||
"it may be necessary to fine-tune the "
|
||||
"expected_avg_time = {}".format(
|
||||
-diff, params.expected_avg_time))
|
||||
print(
|
||||
" !! WARNING !! Performance has improved by {}, "
|
||||
"it may be necessary to fine-tune the "
|
||||
"expected_avg_time = {}".format(-diff, params.expected_avg_time)
|
||||
)
|
||||
|
||||
assert ok, " !! ERROR !! Regression detected"
|
||||
|
||||
@@ -10,21 +10,20 @@ from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", ["Qwen/Qwen2.5-1.5B-Instruct"])
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="This test needs a TPU")
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(), reason="This test needs a TPU")
|
||||
def test_sampler_different(model_name: str):
|
||||
"""
|
||||
Test significantly different sampling params to assert the model produces
|
||||
Test significantly different sampling params to assert the model produces
|
||||
different results.
|
||||
"""
|
||||
llm = LLM(model_name,
|
||||
enforce_eager=False,
|
||||
max_num_seqs=1,
|
||||
max_model_len=512,
|
||||
max_num_batched_tokens=256)
|
||||
prompts = [
|
||||
"Write a short story about a robot that dreams for the first time."
|
||||
]
|
||||
llm = LLM(
|
||||
model_name,
|
||||
enforce_eager=False,
|
||||
max_num_seqs=1,
|
||||
max_model_len=512,
|
||||
max_num_batched_tokens=256,
|
||||
)
|
||||
prompts = ["Write a short story about a robot that dreams for the first time."]
|
||||
sampling_params = SamplingParams(temperature=0.9, min_p=0.2, max_tokens=64)
|
||||
output = llm.generate(prompts, sampling_params)
|
||||
|
||||
@@ -47,7 +46,9 @@ def test_sampler_different(model_name: str):
|
||||
max_tokens=64,
|
||||
# Vary number of ks
|
||||
top_k=random.randint(4, 12),
|
||||
top_p=random.random()) for _ in range(B)
|
||||
top_p=random.random(),
|
||||
)
|
||||
for _ in range(B)
|
||||
]
|
||||
# Make sure first two reqs have the same K/P
|
||||
sampling_params[0] = sampling_params[1]
|
||||
@@ -61,20 +62,18 @@ def test_sampler_different(model_name: str):
|
||||
@pytest.mark.parametrize("model_name", ["Qwen/Qwen2.5-1.5B-Instruct"])
|
||||
# TODO TPU will appear busy if we fan-out test params here
|
||||
@pytest.mark.parametrize("n_prompts", [1])
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="This test needs a TPU")
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(), reason="This test needs a TPU")
|
||||
def test_logprobs(model_name: str, n_prompts: int):
|
||||
"""
|
||||
Request top logprobs with different sampling settings and check
|
||||
that results contains the requested number, ordered ascendingly.
|
||||
that results contains the requested number, ordered ascendingly.
|
||||
"""
|
||||
|
||||
def check_num_logprobs(logprobs, expected_num: int):
|
||||
for step in logprobs:
|
||||
prev_logp = 1.0
|
||||
# order by rank
|
||||
sorted_step = dict(
|
||||
sorted(step.items(), key=lambda item: item[1].rank))
|
||||
sorted_step = dict(sorted(step.items(), key=lambda item: item[1].rank))
|
||||
|
||||
# Can contain the sampled token
|
||||
assert len(step) == expected_num or len(step) == expected_num + 1
|
||||
@@ -84,23 +83,23 @@ def test_logprobs(model_name: str, n_prompts: int):
|
||||
prev_logp = logp.logprob
|
||||
assert logp.rank == rankno + 1
|
||||
|
||||
llm = LLM(model_name,
|
||||
enforce_eager=False,
|
||||
max_num_seqs=1,
|
||||
max_model_len=128,
|
||||
max_num_batched_tokens=128)
|
||||
llm = LLM(
|
||||
model_name,
|
||||
enforce_eager=False,
|
||||
max_num_seqs=1,
|
||||
max_model_len=128,
|
||||
max_num_batched_tokens=128,
|
||||
)
|
||||
prompts = [
|
||||
"Write a short story about a robot that dreams for the first time."
|
||||
] * n_prompts
|
||||
greedy_sampling_params = SamplingParams(temperature=0.0, max_tokens=64,\
|
||||
logprobs=4)
|
||||
regular_sampling_params = SamplingParams(temperature=0.4, max_tokens=64,\
|
||||
logprobs=4)
|
||||
topkp_sampling_params = SamplingParams(temperature=0.4, max_tokens=64,\
|
||||
logprobs=4, top_k=12, top_p=0.5)
|
||||
greedy_sampling_params = SamplingParams(temperature=0.0, max_tokens=64, logprobs=4)
|
||||
regular_sampling_params = SamplingParams(temperature=0.4, max_tokens=64, logprobs=4)
|
||||
topkp_sampling_params = SamplingParams(
|
||||
temperature=0.4, max_tokens=64, logprobs=4, top_k=12, top_p=0.5
|
||||
)
|
||||
|
||||
for sp in [greedy_sampling_params, regular_sampling_params, \
|
||||
topkp_sampling_params]:
|
||||
for sp in [greedy_sampling_params, regular_sampling_params, topkp_sampling_params]:
|
||||
output = llm.generate(prompts, sp)
|
||||
for o in output:
|
||||
check_num_logprobs(o.outputs[0].logprobs, 4)
|
||||
|
||||
@@ -9,14 +9,18 @@ import torch_xla.distributed.spmd as xs
|
||||
import torch_xla.runtime as xr
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
|
||||
init_distributed_environment)
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.model_loader.tpu import TPUModelLoader
|
||||
|
||||
|
||||
def _setup_environment(model):
|
||||
engine_args = EngineArgs(model=model, )
|
||||
engine_args = EngineArgs(
|
||||
model=model,
|
||||
)
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
temp_file = tempfile.mkstemp()[1]
|
||||
@@ -25,7 +29,8 @@ def _setup_environment(model):
|
||||
0,
|
||||
local_rank=0,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
backend="gloo")
|
||||
backend="gloo",
|
||||
)
|
||||
# Under single worker mode, full model is init first and then
|
||||
# partitioned using GSPMD.
|
||||
ensure_model_parallel_initialized(1, 1)
|
||||
@@ -42,7 +47,7 @@ def _get_spmd_mesh():
|
||||
num_devices = xr.global_runtime_device_count()
|
||||
mesh_shape = (num_devices, 1)
|
||||
device_ids = np.array(range(num_devices))
|
||||
MESH = xs.Mesh(device_ids, mesh_shape, ('x', 'y'))
|
||||
MESH = xs.Mesh(device_ids, mesh_shape, ("x", "y"))
|
||||
return MESH
|
||||
|
||||
|
||||
@@ -53,15 +58,17 @@ def _get_spmd_mesh():
|
||||
# Skip large models due to CI runner disk space limitations
|
||||
# "meta-llama/Llama-3.1-8B-Instruct",
|
||||
# "meta-llama/Llama-3.1-70B-Instruct",
|
||||
])
|
||||
],
|
||||
)
|
||||
def test_tpu_model_loader(model):
|
||||
# Skip the 70B test if there are less than 8 chips
|
||||
# TODO: Query using torch xla API, the query API is not working
|
||||
# with SPMD now. However, This test is running under SPMD mode.
|
||||
if '70B' in model and xr.global_runtime_device_count() < 8:
|
||||
if "70B" in model and xr.global_runtime_device_count() < 8:
|
||||
pytest.skip(
|
||||
"Skipping 70B model if the TPU VM has less than 8 chips to \
|
||||
avoid OOM.")
|
||||
avoid OOM."
|
||||
)
|
||||
|
||||
vllm_config = _setup_environment(model)
|
||||
loader = TPUModelLoader(load_config=vllm_config.load_config)
|
||||
|
||||
@@ -10,8 +10,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||
|
||||
# isort: off
|
||||
from vllm.v1.sample.tpu.sampler import (apply_top_k_top_p as
|
||||
apply_top_k_top_p_tpu)
|
||||
from vllm.v1.sample.tpu.sampler import apply_top_k_top_p as apply_top_k_top_p_tpu
|
||||
# isort: on
|
||||
|
||||
if not current_platform.is_tpu():
|
||||
@@ -30,11 +29,10 @@ def test_topk_equivalence_to_native_impl():
|
||||
logits = torch.rand((BATCH_SIZE, VOCAB_SIZE))
|
||||
|
||||
# Random top-k values between 1 and 10.
|
||||
k = torch.randint(1, 10, (BATCH_SIZE, ))
|
||||
k = torch.randint(1, 10, (BATCH_SIZE,))
|
||||
|
||||
# Set k=vocab_size for ~50% of requests in the batch (top-k disabled).
|
||||
k.masked_fill_(torch.randint(0, 2, (BATCH_SIZE, ), dtype=bool),
|
||||
VOCAB_SIZE)
|
||||
k.masked_fill_(torch.randint(0, 2, (BATCH_SIZE,), dtype=bool), VOCAB_SIZE)
|
||||
|
||||
result_tpu = apply_top_k_top_p_tpu(logits=logits.clone(), k=k, p=None)
|
||||
|
||||
@@ -50,15 +48,13 @@ def test_topp_result_sums_past_p():
|
||||
probs = logits.softmax(dim=-1)
|
||||
|
||||
# Random top-p values between 0 and 1.
|
||||
p = torch.rand((BATCH_SIZE, ))
|
||||
p = torch.rand((BATCH_SIZE,))
|
||||
|
||||
# Set p=1 for ~50% of requests in the batch (top-p disabled).
|
||||
p.masked_fill_(torch.randint(0, 2, (BATCH_SIZE, ), dtype=bool), 1)
|
||||
p.masked_fill_(torch.randint(0, 2, (BATCH_SIZE,), dtype=bool), 1)
|
||||
|
||||
no_op_k = torch.tensor([VOCAB_SIZE])
|
||||
logits_masked = apply_top_k_top_p_tpu(logits=logits.clone(),
|
||||
k=no_op_k,
|
||||
p=p)
|
||||
logits_masked = apply_top_k_top_p_tpu(logits=logits.clone(), k=no_op_k, p=p)
|
||||
|
||||
# Verify that the masked logit's probability sums to at least p.
|
||||
probs.masked_fill_(logits_masked.isinf(), 0)
|
||||
@@ -72,16 +68,16 @@ def test_topp_result_sums_past_p():
|
||||
|
||||
def test_topp_basic():
|
||||
with torch.device(xm.xla_device()):
|
||||
logits = torch.tensor([[math.log(0.2),
|
||||
math.log(0.3),
|
||||
math.log(0.5)],
|
||||
[math.log(0.5),
|
||||
math.log(0.1),
|
||||
math.log(0.4)]])
|
||||
logits = torch.tensor(
|
||||
[
|
||||
[math.log(0.2), math.log(0.3), math.log(0.5)],
|
||||
[math.log(0.5), math.log(0.1), math.log(0.4)],
|
||||
]
|
||||
)
|
||||
|
||||
result = apply_top_k_top_p_tpu(logits=logits.clone(),
|
||||
k=torch.tensor([3, 3]),
|
||||
p=torch.tensor([0.79, 0.79]))
|
||||
result = apply_top_k_top_p_tpu(
|
||||
logits=logits.clone(), k=torch.tensor([3, 3]), p=torch.tensor([0.79, 0.79])
|
||||
)
|
||||
|
||||
torch_xla.sync()
|
||||
|
||||
@@ -94,16 +90,16 @@ def test_topp_basic():
|
||||
|
||||
def test_topp_select_all():
|
||||
with torch.device(xm.xla_device()):
|
||||
logits = torch.tensor([[math.log(0.2),
|
||||
math.log(0.3),
|
||||
math.log(0.5)],
|
||||
[math.log(0.5),
|
||||
math.log(0.1),
|
||||
math.log(0.4)]])
|
||||
logits = torch.tensor(
|
||||
[
|
||||
[math.log(0.2), math.log(0.3), math.log(0.5)],
|
||||
[math.log(0.5), math.log(0.1), math.log(0.4)],
|
||||
]
|
||||
)
|
||||
|
||||
result = apply_top_k_top_p_tpu(logits=logits.clone(),
|
||||
k=torch.tensor([3, 3]),
|
||||
p=torch.tensor([1.0, 1.0]))
|
||||
result = apply_top_k_top_p_tpu(
|
||||
logits=logits.clone(), k=torch.tensor([3, 3]), p=torch.tensor([1.0, 1.0])
|
||||
)
|
||||
|
||||
torch_xla.sync()
|
||||
|
||||
@@ -114,14 +110,12 @@ def test_topp_with_ties():
|
||||
with torch.device(xm.xla_device()):
|
||||
# Input has multiple math.log(0.3).
|
||||
logits = torch.tensor(
|
||||
[[math.log(0.3),
|
||||
math.log(0.3),
|
||||
math.log(0.3),
|
||||
math.log(0.1)]])
|
||||
[[math.log(0.3), math.log(0.3), math.log(0.3), math.log(0.1)]]
|
||||
)
|
||||
|
||||
result = apply_top_k_top_p_tpu(logits=logits.clone(),
|
||||
k=torch.tensor([4]),
|
||||
p=torch.tensor([0.2]))
|
||||
result = apply_top_k_top_p_tpu(
|
||||
logits=logits.clone(), k=torch.tensor([4]), p=torch.tensor([0.2])
|
||||
)
|
||||
|
||||
torch_xla.sync()
|
||||
|
||||
@@ -135,17 +129,17 @@ def test_topp_with_ties():
|
||||
|
||||
def test_both_topk_topp():
|
||||
with torch.device(xm.xla_device()):
|
||||
logits = torch.tensor([[math.log(0.2),
|
||||
math.log(0.3),
|
||||
math.log(0.5)],
|
||||
[math.log(0.5),
|
||||
math.log(0.1),
|
||||
math.log(0.4)]])
|
||||
logits = torch.tensor(
|
||||
[
|
||||
[math.log(0.2), math.log(0.3), math.log(0.5)],
|
||||
[math.log(0.5), math.log(0.1), math.log(0.4)],
|
||||
]
|
||||
)
|
||||
|
||||
# Set k=1 for the first batch.
|
||||
result = apply_top_k_top_p_tpu(logits=logits.clone(),
|
||||
k=torch.tensor([1, 3]),
|
||||
p=torch.tensor([0.79, 0.79]))
|
||||
result = apply_top_k_top_p_tpu(
|
||||
logits=logits.clone(), k=torch.tensor([1, 3]), p=torch.tensor([0.79, 0.79])
|
||||
)
|
||||
|
||||
torch_xla.sync()
|
||||
|
||||
|
||||
@@ -4,11 +4,11 @@
|
||||
|
||||
Run `pytest tests/quantization/test_tpu_int8.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.tpu_int8 import (
|
||||
TPUInt8LinearMethod)
|
||||
from vllm.model_executor.layers.quantization.tpu_int8 import TPUInt8LinearMethod
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...models.registry import HF_EXAMPLE_MODELS
|
||||
@@ -16,8 +16,9 @@ from ...models.registry import HF_EXAMPLE_MODELS
|
||||
MODELS = ["Qwen/Qwen2.5-0.5B-Instruct"]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_tpu(),
|
||||
reason="TPU Int8 is only enabled for TPUs.")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_tpu(), reason="TPU Int8 is only enabled for TPUs."
|
||||
)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [10])
|
||||
@@ -26,20 +27,28 @@ MODELS = ["Qwen/Qwen2.5-0.5B-Instruct"]
|
||||
[
|
||||
# w8a8 dynamic activation
|
||||
{
|
||||
'quantization_config': {
|
||||
'quant_method': 'tpu_int8',
|
||||
'activation_scheme': 'dynamic'
|
||||
"quantization_config": {
|
||||
"quant_method": "tpu_int8",
|
||||
"activation_scheme": "dynamic",
|
||||
}
|
||||
}
|
||||
])
|
||||
def test_model_tpu_int8(vllm_runner, model: str, dtype: str, max_tokens: int,
|
||||
hf_overrides: dict, monkeypatch) -> None:
|
||||
],
|
||||
)
|
||||
def test_model_tpu_int8(
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
hf_overrides: dict,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
activation_scheme = hf_overrides.get('quantization_config',
|
||||
{}).get('activation_scheme')
|
||||
quantize_activation = activation_scheme == 'dynamic'
|
||||
activation_scheme = hf_overrides.get("quantization_config", {}).get(
|
||||
"activation_scheme"
|
||||
)
|
||||
quantize_activation = activation_scheme == "dynamic"
|
||||
|
||||
# Allows using apply_model
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
@@ -9,8 +9,10 @@ import torch_xla.distributed.spmd as xs
|
||||
import torch_xla.runtime as xr
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
|
||||
init_distributed_environment)
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.distributed.tpu_distributed_utils import XlaQKVParallelLinear
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
@@ -36,7 +38,8 @@ def setup_environment():
|
||||
0,
|
||||
local_rank=0,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
backend="gloo")
|
||||
backend="gloo",
|
||||
)
|
||||
ensure_model_parallel_initialized(1, 1)
|
||||
yield
|
||||
|
||||
@@ -51,7 +54,7 @@ def _get_spmd_mesh():
|
||||
num_devices = xr.global_runtime_device_count()
|
||||
mesh_shape = (num_devices, 1)
|
||||
device_ids = np.array(range(num_devices))
|
||||
MESH = xs.Mesh(device_ids, mesh_shape, ('x', 'y'))
|
||||
MESH = xs.Mesh(device_ids, mesh_shape, ("x", "y"))
|
||||
return MESH
|
||||
|
||||
|
||||
@@ -59,7 +62,7 @@ def _get_spmd_mesh():
|
||||
# `xr.use_spmd()` will set a global state, and this state is not reversible.
|
||||
# Therefore, non-SPMD tests should be run before SPMD tests.
|
||||
@pytest.mark.parametrize("mesh", [None, _get_spmd_mesh()])
|
||||
@pytest.mark.parametrize("device", ['cpu', 'xla'])
|
||||
@pytest.mark.parametrize("device", ["cpu", "xla"])
|
||||
@torch.no_grad()
|
||||
def test_xla_qkv_linear(bias, mesh, device):
|
||||
torch.manual_seed(123)
|
||||
|
||||
@@ -4,18 +4,25 @@
|
||||
import pytest
|
||||
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.config import (CacheConfig, ModelConfig, SchedulerConfig, VllmConfig,
|
||||
set_current_vllm_config)
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import GiB_bytes
|
||||
from vllm.v1.core.kv_cache_utils import (estimate_max_model_len,
|
||||
get_kv_cache_configs)
|
||||
from vllm.v1.core.sched.output import (CachedRequestData, NewRequestData,
|
||||
SchedulerOutput)
|
||||
from vllm.v1.core.kv_cache_utils import estimate_max_model_len, get_kv_cache_configs
|
||||
from vllm.v1.core.sched.output import CachedRequestData, NewRequestData, SchedulerOutput
|
||||
from vllm.v1.worker.tpu_model_runner import (
|
||||
TPUModelRunner, _get_padded_num_reqs_with_upper_limit,
|
||||
_get_padded_token_len, _get_req_paddings, _get_token_paddings)
|
||||
TPUModelRunner,
|
||||
_get_padded_num_reqs_with_upper_limit,
|
||||
_get_padded_token_len,
|
||||
_get_req_paddings,
|
||||
_get_token_paddings,
|
||||
)
|
||||
|
||||
|
||||
def get_vllm_config():
|
||||
@@ -67,10 +74,11 @@ def _schedule_new_request(*req_ids: str) -> SchedulerOutput:
|
||||
mm_features=[],
|
||||
sampling_params=SamplingParams(),
|
||||
pooling_params=PoolingParams(),
|
||||
block_ids=([0], ), # block_ids should be tuple[list[int]]
|
||||
block_ids=([0],), # block_ids should be tuple[list[int]]
|
||||
num_computed_tokens=0,
|
||||
lora_request=None,
|
||||
))
|
||||
)
|
||||
)
|
||||
num_scheduled_tokens[req_id] = 3
|
||||
total_num_scheduled_tokens += num_scheduled_tokens[req_id]
|
||||
|
||||
@@ -99,7 +107,7 @@ def _is_req_added(model_runner, req_id: str) -> bool:
|
||||
|
||||
def _is_req_state_block_table_match(model_runner, req_id: str) -> bool:
|
||||
"""Check if the request state block IDs match the block table.
|
||||
|
||||
|
||||
This function handles both legacy BlockTable and new MultiGroupBlockTable
|
||||
structures for backward compatibility.
|
||||
"""
|
||||
@@ -206,7 +214,7 @@ def test_update_states_request_resumed(model_runner):
|
||||
req_ids=[req_id],
|
||||
resumed_from_preemption=[False],
|
||||
new_token_ids=[[]],
|
||||
new_block_ids=[([], )],
|
||||
new_block_ids=[([],)],
|
||||
num_computed_tokens=[0],
|
||||
)
|
||||
|
||||
@@ -303,27 +311,23 @@ def test_get_paddings():
|
||||
# Bucketed padding
|
||||
min_token_size, max_token_size, padding_gap = 16, 512, 64
|
||||
expected_paddings = [16, 32, 64, 128, 192, 256, 320, 384, 448, 512]
|
||||
actual_paddings = _get_token_paddings(min_token_size, max_token_size,
|
||||
padding_gap)
|
||||
actual_paddings = _get_token_paddings(min_token_size, max_token_size, padding_gap)
|
||||
|
||||
# Bucketed padding with max_token_size not a power of two.
|
||||
max_token_size = 317
|
||||
expected_paddings = [16, 32, 64, 128, 192, 256, 320]
|
||||
actual_paddings = _get_token_paddings(min_token_size, max_token_size,
|
||||
padding_gap)
|
||||
actual_paddings = _get_token_paddings(min_token_size, max_token_size, padding_gap)
|
||||
assert actual_paddings == expected_paddings
|
||||
|
||||
# Exponential padding.
|
||||
max_token_size, padding_gap = 1024, 0
|
||||
expected_paddings = [16, 32, 64, 128, 256, 512, 1024]
|
||||
actual_paddings = _get_token_paddings(min_token_size, max_token_size,
|
||||
padding_gap)
|
||||
actual_paddings = _get_token_paddings(min_token_size, max_token_size, padding_gap)
|
||||
assert actual_paddings == expected_paddings
|
||||
# Exponential padding with max_token_size not a power of two.
|
||||
max_token_size = 317
|
||||
expected_paddings = [16, 32, 64, 128, 256, 512]
|
||||
actual_paddings = _get_token_paddings(min_token_size, max_token_size,
|
||||
padding_gap)
|
||||
actual_paddings = _get_token_paddings(min_token_size, max_token_size, padding_gap)
|
||||
assert actual_paddings == expected_paddings
|
||||
|
||||
|
||||
@@ -350,32 +354,31 @@ def test_get_req_paddings():
|
||||
assert _get_req_paddings(8, 36) == [8, 16, 32, 36]
|
||||
|
||||
|
||||
def test_init_kv_cache_with_kv_sharing_invalid_target_layer_order(
|
||||
model_runner):
|
||||
def test_init_kv_cache_with_kv_sharing_invalid_target_layer_order(model_runner):
|
||||
layer_0 = "model.layers.0.self_attn.attn"
|
||||
layer_1 = "model.layers.1.self_attn.attn"
|
||||
error_msg = f"{layer_1} must come before the current layer"
|
||||
vllm_config = model_runner.vllm_config
|
||||
with pytest.raises(ValueError, match=error_msg), \
|
||||
set_current_vllm_config(vllm_config):
|
||||
with (
|
||||
pytest.raises(ValueError, match=error_msg),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
fwd_context = {
|
||||
# initialization below will fail because target layer is invalid;
|
||||
# the target layer needs to come before layer 1
|
||||
layer_0:
|
||||
Attention(
|
||||
layer_0: Attention(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=1.0,
|
||||
prefix=layer_0,
|
||||
kv_sharing_target_layer_name=layer_1,
|
||||
),
|
||||
layer_1:
|
||||
Attention(
|
||||
layer_1: Attention(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=1.0,
|
||||
prefix=layer_1,
|
||||
)
|
||||
),
|
||||
}
|
||||
# suppress var not used error
|
||||
assert fwd_context is not None
|
||||
@@ -387,25 +390,25 @@ def test_init_kv_cache_with_kv_sharing_target_layer_not_exist(model_runner):
|
||||
invalid_layer = "model.layers.0.cross_attn.attn"
|
||||
error_msg = f"{invalid_layer} is not a valid Attention layer in the model"
|
||||
vllm_config = model_runner.vllm_config
|
||||
with pytest.raises(ValueError, match=error_msg), \
|
||||
set_current_vllm_config(vllm_config):
|
||||
with (
|
||||
pytest.raises(ValueError, match=error_msg),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
fwd_context = {
|
||||
layer_0:
|
||||
Attention(
|
||||
layer_0: Attention(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=1.0,
|
||||
prefix=layer_0,
|
||||
),
|
||||
layer_1:
|
||||
Attention(
|
||||
layer_1: Attention(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=1.0,
|
||||
prefix=layer_1,
|
||||
# invalid layer: cross_attn.atn doesn't exist!
|
||||
kv_sharing_target_layer_name=invalid_layer,
|
||||
)
|
||||
),
|
||||
}
|
||||
# suppress var not used error
|
||||
assert fwd_context is not None
|
||||
@@ -416,26 +419,26 @@ def test_init_kv_cache_with_kv_sharing_target_same_as_current(model_runner):
|
||||
layer_1 = "model.layers.1.self_attn.attn"
|
||||
error_msg = f"{layer_1} cannot be the same as the current layer"
|
||||
vllm_config = model_runner.vllm_config
|
||||
with pytest.raises(ValueError, match=error_msg), \
|
||||
set_current_vllm_config(vllm_config):
|
||||
with (
|
||||
pytest.raises(ValueError, match=error_msg),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
fwd_context = {
|
||||
# initialization below will fail because target layer is invalid;
|
||||
# the target layer needs to come before layer 1
|
||||
layer_0:
|
||||
Attention(
|
||||
layer_0: Attention(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=1.0,
|
||||
prefix=layer_0,
|
||||
),
|
||||
layer_1:
|
||||
Attention(
|
||||
layer_1: Attention(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=1.0,
|
||||
prefix=layer_1,
|
||||
kv_sharing_target_layer_name=layer_1,
|
||||
)
|
||||
),
|
||||
}
|
||||
# suppress var not used error
|
||||
assert fwd_context is not None
|
||||
@@ -447,20 +450,18 @@ def test_init_kv_cache_without_kv_sharing():
|
||||
vllm_config = get_vllm_config()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
fwd_context = {
|
||||
layer_0:
|
||||
Attention(
|
||||
layer_0: Attention(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=1.0,
|
||||
prefix=layer_0,
|
||||
),
|
||||
layer_1:
|
||||
Attention(
|
||||
layer_1: Attention(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=1.0,
|
||||
prefix=layer_1,
|
||||
)
|
||||
),
|
||||
}
|
||||
# suppress var not used error
|
||||
assert fwd_context is not None
|
||||
@@ -475,17 +476,17 @@ def test_init_kv_cache_without_kv_sharing():
|
||||
available_memory = 20 * GiB_bytes
|
||||
# page size for each layer KV can be calculated as
|
||||
# 2 (non-MLA) * 8 (num_heads) * 128 (head_dim)
|
||||
# * 2 (bfloat16, kv_cache dtype) * 128 (block_size) = 512KB
|
||||
# * 2 (bfloat16, kv_cache dtype) * 128 (block_size) = 512KB
|
||||
num_expected_blocks = 20480 # 20GB / 512KB / 2 (num layers)
|
||||
kv_cache_config = get_kv_cache_configs(vllm_config, [kv_cache_spec],
|
||||
[available_memory])[0]
|
||||
kv_cache_config = get_kv_cache_configs(
|
||||
vllm_config, [kv_cache_spec], [available_memory]
|
||||
)[0]
|
||||
assert kv_cache_config.num_blocks == num_expected_blocks
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 2
|
||||
assert kv_cache_config.kv_cache_tensors[0].size == available_memory // 2
|
||||
assert kv_cache_config.kv_cache_tensors[1].size == available_memory // 2
|
||||
|
||||
max_context_len =\
|
||||
estimate_max_model_len(vllm_config, kv_cache_spec, 5 * GiB_bytes)
|
||||
max_context_len = estimate_max_model_len(vllm_config, kv_cache_spec, 5 * GiB_bytes)
|
||||
# max context len with KV sharing should be 2x as large as without
|
||||
# max_context_len = available_memory / (page_size / block_size) / num_caches
|
||||
# max_context_len = 5GB / (512KB / 128) / 2 = 655360
|
||||
@@ -495,8 +496,9 @@ def test_init_kv_cache_without_kv_sharing():
|
||||
# this will only allocate 2 block worth of memory (2 * 512kb)
|
||||
kv_cache_config.num_blocks = 1
|
||||
for kv_cache_tensor in kv_cache_config.kv_cache_tensors:
|
||||
kv_cache_tensor.size = (
|
||||
kv_cache_spec[kv_cache_tensor.shared_by[0]].page_size_bytes)
|
||||
kv_cache_tensor.size = kv_cache_spec[
|
||||
kv_cache_tensor.shared_by[0]
|
||||
].page_size_bytes
|
||||
|
||||
model_runner.initialize_kv_cache(kv_cache_config)
|
||||
|
||||
@@ -518,21 +520,19 @@ def test_init_kv_cache_with_kv_sharing_valid():
|
||||
vllm_config = get_vllm_config()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
fwd_context = {
|
||||
layer_0:
|
||||
Attention(
|
||||
layer_0: Attention(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=1.0,
|
||||
prefix=layer_0,
|
||||
),
|
||||
layer_1:
|
||||
Attention(
|
||||
layer_1: Attention(
|
||||
num_heads=8,
|
||||
head_size=128,
|
||||
scale=1.0,
|
||||
prefix=layer_1,
|
||||
kv_sharing_target_layer_name="model.layers.0.self_attn.attn",
|
||||
)
|
||||
),
|
||||
}
|
||||
# suppress var not used error
|
||||
assert fwd_context is not None
|
||||
@@ -550,24 +550,23 @@ def test_init_kv_cache_with_kv_sharing_valid():
|
||||
# with KV sharing, we can allocate (available_mem//page_size//1) blocks
|
||||
# which is twice as many as without KV sharing
|
||||
num_expected_blocks = 2 * 20480 # 20GB / 512KB
|
||||
kv_cache_config = get_kv_cache_configs(vllm_config, [kv_cache_spec],
|
||||
[available_memory])[0]
|
||||
kv_cache_config = get_kv_cache_configs(
|
||||
vllm_config, [kv_cache_spec], [available_memory]
|
||||
)[0]
|
||||
assert kv_cache_config.num_blocks == num_expected_blocks
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 1
|
||||
# Each layer now has twice the available memory for KV cache
|
||||
# compared to no KV sharing
|
||||
assert kv_cache_config.kv_cache_tensors[0].size == available_memory
|
||||
|
||||
max_context_len =\
|
||||
estimate_max_model_len(vllm_config, kv_cache_spec, 5 * GiB_bytes)
|
||||
max_context_len = estimate_max_model_len(vllm_config, kv_cache_spec, 5 * GiB_bytes)
|
||||
# max context len with KV sharing should be 2x as large as without
|
||||
assert max_context_len == (2 * 655360)
|
||||
|
||||
# important: override tensor size to prevent large mem alloc during test
|
||||
# this will only allocate 1 block worth of memory (512kb)
|
||||
kv_cache_config.num_blocks = 1
|
||||
kv_cache_config.kv_cache_tensors[0].size =\
|
||||
kv_cache_spec[layer_0].page_size_bytes
|
||||
kv_cache_config.kv_cache_tensors[0].size = kv_cache_spec[layer_0].page_size_bytes
|
||||
|
||||
model_runner.initialize_kv_cache(kv_cache_config)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user