[NIXL][1/N] Refactor kernel_block_size detection (#35752)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -9,7 +9,7 @@ import textwrap
|
||||
import time
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import msgspec
|
||||
@@ -332,14 +332,22 @@ def test_kv_transfer_handshake(dist_init):
|
||||
|
||||
# Prefill connector will register KV cache to populate proper handshake
|
||||
# metadata.
|
||||
# TODO this must match with values used in kv cache config
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2)
|
||||
prefill_connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
kv_cache_spec = cast(
|
||||
AttentionSpec, kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
)
|
||||
kv_cache_shape = FlashAttentionBackend.get_kv_cache_shape(
|
||||
num_blocks=2, block_size=16, num_kv_heads=4, head_size=64
|
||||
num_blocks=kv_cache_config.num_blocks,
|
||||
block_size=kv_cache_spec.block_size,
|
||||
num_kv_heads=kv_cache_spec.num_kv_heads,
|
||||
head_size=kv_cache_spec.head_size,
|
||||
)
|
||||
shared_tensor = torch.zeros(*kv_cache_shape, dtype=torch.float16)
|
||||
unique_tensor = torch.zeros(*kv_cache_shape, dtype=torch.float16)
|
||||
shared_tensor = torch.zeros(*kv_cache_shape, dtype=kv_cache_spec.dtype)
|
||||
unique_tensor = torch.zeros(*kv_cache_shape, dtype=kv_cache_spec.dtype)
|
||||
kv_caches = {
|
||||
"layer0": shared_tensor,
|
||||
"layer1": unique_tensor,
|
||||
@@ -383,7 +391,7 @@ def test_kv_transfer_handshake(dist_init):
|
||||
|
||||
# Decode connector will be able to create handshake with the prefill connector.
|
||||
decode_connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
decode_connector.register_kv_caches(kv_caches)
|
||||
|
||||
@@ -525,11 +533,13 @@ class TestNixlHandshake:
|
||||
request_id = "req_id"
|
||||
|
||||
# Test worker role in decode server.
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2)
|
||||
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
vllm_config,
|
||||
connector.engine_id,
|
||||
hand_shake_latency=0,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
assert isinstance(connector.connector_worker.nixl_wrapper, FakeNixlWrapper)
|
||||
worker = connector.connector_worker
|
||||
@@ -1479,18 +1489,22 @@ def test_register_kv_caches(
|
||||
patch(f"{nixl_module}.threading.Event"),
|
||||
patch(f"{nixl_module}.threading.Thread") as mock_thread,
|
||||
patch(f"{nixl_module}.get_current_attn_backend") as mock_get_attn_backend,
|
||||
patch(f"{nixl_module}.get_current_attn_backends") as mock_get_attn_backends,
|
||||
):
|
||||
# Ensure get_attn_backend returns the correct value due to
|
||||
# _cached_get_attn_backend returning the backend from previous
|
||||
# test run if not mocking.
|
||||
mock_get_attn_backend.return_value = backend_cls
|
||||
mock_get_attn_backends.return_value = [backend_cls]
|
||||
|
||||
# Create connector
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2)
|
||||
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
vllm_config,
|
||||
connector.engine_id,
|
||||
hand_shake_latency=0,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
|
||||
# Get the mock instance
|
||||
@@ -1515,6 +1529,13 @@ def test_register_kv_caches(
|
||||
num_layers = 32
|
||||
block_size = 16
|
||||
num_blocks = 8
|
||||
# Keep the fake worker's expected num_blocks in sync with the
|
||||
# cross-layer tensor we are about to register.
|
||||
worker_kv_cache_config = make_kv_cache_config(
|
||||
block_size=block_size, num_blocks=num_blocks
|
||||
)
|
||||
connector.connector_worker.kv_cache_config = worker_kv_cache_config
|
||||
connector.connector_worker.num_blocks = worker_kv_cache_config.num_blocks
|
||||
kv_cache_spec = AttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=4,
|
||||
@@ -1568,11 +1589,17 @@ def test_register_kv_caches(
|
||||
|
||||
else:
|
||||
# Create test kv cache tensors using proper backend shape
|
||||
kv_cache_shape = backend_cls.get_kv_cache_shape(
|
||||
num_blocks=2, block_size=16, num_kv_heads=4, head_size=64
|
||||
kv_cache_spec = cast(
|
||||
AttentionSpec, kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
)
|
||||
shared_tensor = torch.zeros(*kv_cache_shape, dtype=torch.float16)
|
||||
unique_tensor = torch.zeros(*kv_cache_shape, dtype=torch.float16)
|
||||
kv_cache_shape = backend_cls.get_kv_cache_shape(
|
||||
num_blocks=kv_cache_config.num_blocks,
|
||||
block_size=kv_cache_spec.block_size,
|
||||
num_kv_heads=kv_cache_spec.num_kv_heads,
|
||||
head_size=kv_cache_spec.head_size,
|
||||
)
|
||||
shared_tensor = torch.zeros(*kv_cache_shape, dtype=kv_cache_spec.dtype)
|
||||
unique_tensor = torch.zeros(*kv_cache_shape, dtype=kv_cache_spec.dtype)
|
||||
kv_caches = {
|
||||
"layer0": shared_tensor,
|
||||
"layer1": unique_tensor,
|
||||
@@ -1606,7 +1633,7 @@ def test_register_kv_caches(
|
||||
unique_tensor[1].data_ptr(),
|
||||
]
|
||||
expected_num_entries = 4
|
||||
expected_blocks_count = 8
|
||||
expected_blocks_count = kv_cache_config.num_blocks * 4
|
||||
|
||||
# Execute register_kv_caches
|
||||
connector.register_kv_caches(kv_caches)
|
||||
@@ -1639,7 +1666,7 @@ def test_register_kv_caches(
|
||||
num_blocks = 8
|
||||
expected_block_len = expected_tensor_size // num_blocks
|
||||
else:
|
||||
num_blocks = 2
|
||||
num_blocks = kv_cache_config.num_blocks
|
||||
if is_blocks_first:
|
||||
expected_block_len = expected_tensor_size // num_blocks // 2
|
||||
else:
|
||||
@@ -2226,15 +2253,22 @@ def test_compatibility_hash_validation(
|
||||
"enforce_handshake_compat": enforce_handshake_compat
|
||||
},
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2)
|
||||
decode_connector = NixlConnector(
|
||||
local_vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
local_vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
decode_worker = decode_connector.connector_worker
|
||||
kv_cache_shape = decode_worker.attn_backend.get_kv_cache_shape(
|
||||
num_blocks=2, block_size=16, num_kv_heads=4, head_size=64
|
||||
kv_cache_spec = cast(
|
||||
AttentionSpec, kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
)
|
||||
shared_tensor = torch.zeros(*kv_cache_shape, dtype=torch.float16)
|
||||
unique_tensor = torch.zeros(*kv_cache_shape, dtype=torch.float16)
|
||||
kv_cache_shape = decode_worker.attn_backend.get_kv_cache_shape(
|
||||
num_blocks=kv_cache_config.num_blocks,
|
||||
block_size=kv_cache_spec.block_size,
|
||||
num_kv_heads=kv_cache_spec.num_kv_heads,
|
||||
head_size=kv_cache_spec.head_size,
|
||||
)
|
||||
shared_tensor = torch.zeros(*kv_cache_shape, dtype=kv_cache_spec.dtype)
|
||||
unique_tensor = torch.zeros(*kv_cache_shape, dtype=kv_cache_spec.dtype)
|
||||
kv_caches = {
|
||||
"layer0": shared_tensor,
|
||||
"layer1": unique_tensor,
|
||||
|
||||
@@ -38,7 +38,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.worker.gpu_input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.utils import AttentionGroup, select_common_block_size
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
NUM_BLOCKS = 10
|
||||
@@ -203,37 +203,25 @@ def _make_kv_cache_spec() -> FullAttentionSpec:
|
||||
def test_select_common_block_size_prefers_manager_block_size():
|
||||
backend_a = _make_mock_backend_for_kernel_block_size([MultipleOf(32)])
|
||||
backend_b = _make_mock_backend_for_kernel_block_size([64, MultipleOf(16)])
|
||||
attn_groups = [
|
||||
AttentionGroup(backend_a, [], [], _make_kv_cache_spec(), 0),
|
||||
AttentionGroup(backend_b, [], [], _make_kv_cache_spec(), 0),
|
||||
]
|
||||
|
||||
selected_size = select_common_block_size(128, attn_groups)
|
||||
selected_size = select_common_block_size(128, [backend_a, backend_b])
|
||||
assert selected_size == 128
|
||||
|
||||
|
||||
def test_select_common_block_size_uses_largest_shared_int():
|
||||
backend_a = _make_mock_backend_for_kernel_block_size([128, 64])
|
||||
backend_b = _make_mock_backend_for_kernel_block_size([64, 32])
|
||||
attn_groups = [
|
||||
AttentionGroup(backend_a, [], [], _make_kv_cache_spec(), 0),
|
||||
AttentionGroup(backend_b, [], [], _make_kv_cache_spec(), 0),
|
||||
]
|
||||
|
||||
selected_size = select_common_block_size(256, attn_groups)
|
||||
selected_size = select_common_block_size(256, [backend_a, backend_b])
|
||||
assert selected_size == 64
|
||||
|
||||
|
||||
def test_select_common_block_size_no_valid_option():
|
||||
backend_a = _make_mock_backend_for_kernel_block_size([64])
|
||||
backend_b = _make_mock_backend_for_kernel_block_size([MultipleOf(16)])
|
||||
attn_groups = [
|
||||
AttentionGroup(backend_a, [], [], _make_kv_cache_spec(), 0),
|
||||
AttentionGroup(backend_b, [], [], _make_kv_cache_spec(), 0),
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
select_common_block_size(48, attn_groups)
|
||||
select_common_block_size(48, [backend_a, backend_b])
|
||||
|
||||
|
||||
def test_update_states_new_request(model_runner, dist_init):
|
||||
|
||||
Reference in New Issue
Block a user