[KV Offload] Deduplicate replicated MLA KV in the shared CPU region (#48906)
Signed-off-by: Change72 <changg@nvidia.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
7a29a3c54c
commit
7eca0e1a64
@@ -41,7 +41,10 @@ NUM_MAPPINGS_PER_GROUP = [2]
|
||||
@pytest.mark.parametrize("num_tensors", NUM_TENSORS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
@pytest.mark.parametrize("use_shared_memory", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("use_shared_memory", "replicated_layout"),
|
||||
[(False, False), (True, False), (True, True)],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_transfer(
|
||||
default_vllm_config,
|
||||
@@ -55,6 +58,7 @@ def test_transfer(
|
||||
seed: int,
|
||||
device: str,
|
||||
use_shared_memory: bool,
|
||||
replicated_layout: bool,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
|
||||
@@ -95,11 +99,15 @@ def test_transfer(
|
||||
gpu_page_size_bytes * num_tensors * blocks_per_chunk,
|
||||
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
|
||||
)
|
||||
simulated_world_size = 2
|
||||
kv_bytes_per_block = (
|
||||
cpu_page_size if replicated_layout else cpu_page_size * simulated_world_size
|
||||
)
|
||||
mmap_region = SharedOffloadRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=num_cpu_blocks,
|
||||
rank=0,
|
||||
kv_bytes_per_block=cpu_page_size,
|
||||
kv_bytes_per_block=kv_bytes_per_block,
|
||||
cpu_page_size=cpu_page_size,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user