[Core][BugFix] Fix PP KV cache sharding memory validation (#33698)
Signed-off-by: junuxyz <216036880+junuxyz@users.noreply.github.com>
This commit is contained in:
@@ -1046,6 +1046,99 @@ def test_get_kv_cache_configs_multiple_workers():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"asymmetric_memory",
|
||||
[False, True],
|
||||
ids=["symmetric", "asymmetric"],
|
||||
)
|
||||
def test_get_kv_cache_configs_pp_sharding(asymmetric_memory):
|
||||
model_config = ModelConfig(max_model_len=512)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
ref_kv_cache_spec = new_kv_cache_spec()
|
||||
pp_kv_cache_specs = [
|
||||
{"layer1": ref_kv_cache_spec},
|
||||
{"layer2": ref_kv_cache_spec},
|
||||
]
|
||||
|
||||
expected_num_blocks = model_config.max_model_len // ref_kv_cache_spec.block_size + 1
|
||||
avail_memory = ref_kv_cache_spec.page_size_bytes * expected_num_blocks
|
||||
|
||||
# With per-worker validation, each worker only needs memory for its own
|
||||
# layers. Worker 2 having more memory shouldn't affect worker 1's config.
|
||||
available_memory = (
|
||||
[avail_memory, avail_memory * 2] if asymmetric_memory else [avail_memory] * 2
|
||||
)
|
||||
|
||||
kv_cache_configs = get_kv_cache_configs(
|
||||
vllm_config,
|
||||
pp_kv_cache_specs,
|
||||
available_memory,
|
||||
)
|
||||
|
||||
assert kv_cache_configs == [
|
||||
KVCacheConfig(
|
||||
num_blocks=expected_num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * expected_num_blocks,
|
||||
shared_by=["layer1"],
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer1"], ref_kv_cache_spec)],
|
||||
),
|
||||
KVCacheConfig(
|
||||
num_blocks=expected_num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=ref_kv_cache_spec.page_size_bytes * expected_num_blocks,
|
||||
shared_by=["layer2"],
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["layer2"], ref_kv_cache_spec)],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def test_project_kv_cache_groups_to_worker():
|
||||
spec_a = new_kv_cache_spec()
|
||||
spec_b = new_kv_cache_spec(num_kv_heads=4)
|
||||
|
||||
global_groups = [
|
||||
KVCacheGroupSpec(["layer1", "layer2", "layer3"], spec_a),
|
||||
]
|
||||
worker_spec = {"layer1": spec_a, "layer2": spec_a}
|
||||
projected = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
global_groups, worker_spec
|
||||
)
|
||||
assert len(projected) == 1
|
||||
assert projected[0].layer_names == ["layer1", "layer2"]
|
||||
assert projected[0].kv_cache_spec is spec_a
|
||||
|
||||
projected = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
global_groups, {"layer4": spec_a}
|
||||
)
|
||||
assert len(projected) == 1
|
||||
assert projected[0].layer_names == []
|
||||
assert projected[0].kv_cache_spec is spec_a
|
||||
|
||||
uniform_spec = UniformTypeKVCacheSpecs(
|
||||
block_size=16,
|
||||
kv_cache_specs={"layer1": spec_a, "layer2": spec_b, "layer3": spec_a},
|
||||
)
|
||||
global_groups_uniform = [
|
||||
KVCacheGroupSpec(["layer1", "layer2", "layer3"], uniform_spec),
|
||||
]
|
||||
projected = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
global_groups_uniform, {"layer1": spec_a, "layer3": spec_a}
|
||||
)
|
||||
assert len(projected) == 1
|
||||
assert projected[0].layer_names == ["layer1", "layer3"]
|
||||
proj_spec = projected[0].kv_cache_spec
|
||||
assert isinstance(proj_spec, UniformTypeKVCacheSpecs)
|
||||
assert set(proj_spec.kv_cache_specs.keys()) == {"layer1", "layer3"}
|
||||
|
||||
|
||||
def test_merge_kv_cache_spec():
|
||||
same_layer_specs = [
|
||||
new_kv_cache_spec(num_kv_heads=32),
|
||||
|
||||
Reference in New Issue
Block a user