[KV Offload] Expose data-parallel topology to offloading backends (#51879)
Signed-off-by: Ziqi Fan <ziqif@nvidia.com>
This commit is contained in:
@@ -326,13 +326,17 @@ def test_dcp_scales_attention_but_not_mamba_group_blocks():
|
||||
] == [1, 3]
|
||||
|
||||
|
||||
def test_preserves_data_parallel_index():
|
||||
def test_preserves_data_parallel_config():
|
||||
config = _make_vllm_config()
|
||||
config.parallel_config.data_parallel_index = 2
|
||||
config.parallel_config.data_parallel_size = 4
|
||||
config.parallel_config.data_parallel_rank_local = 1
|
||||
|
||||
offloading_config = build_offloading_config(config, _make_kv_cache_config())
|
||||
|
||||
assert offloading_config.parallel.data_parallel_index == 2
|
||||
assert offloading_config.parallel.data_parallel_size == 4
|
||||
assert offloading_config.parallel.data_parallel_rank_local == 1
|
||||
|
||||
|
||||
def test_resolves_heterogeneous_hybrid_block_sizes():
|
||||
|
||||
@@ -191,6 +191,8 @@ def _offloading_config(rank: int = 0) -> OffloadingConfig:
|
||||
pcp_size=1,
|
||||
dcp_size=1,
|
||||
data_parallel_index=0,
|
||||
data_parallel_size=1,
|
||||
data_parallel_rank_local=None,
|
||||
is_parallelism_agnostic=False,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -49,6 +49,8 @@ def _make_offloading_config(
|
||||
pcp_size: int = 1,
|
||||
dcp_size: int = 1,
|
||||
data_parallel_index: int = 0,
|
||||
data_parallel_size: int = 1,
|
||||
data_parallel_rank_local: int | None = None,
|
||||
is_parallelism_agnostic: bool = False,
|
||||
replicated_layout: bool = False,
|
||||
extra_config: dict[str, Any] | None = None,
|
||||
@@ -81,6 +83,8 @@ def _make_offloading_config(
|
||||
pcp_size=pcp_size,
|
||||
dcp_size=dcp_size,
|
||||
data_parallel_index=data_parallel_index,
|
||||
data_parallel_size=data_parallel_size,
|
||||
data_parallel_rank_local=data_parallel_rank_local,
|
||||
is_parallelism_agnostic=is_parallelism_agnostic,
|
||||
),
|
||||
replicated_layout=replicated_layout,
|
||||
|
||||
@@ -49,6 +49,8 @@ def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
|
||||
pcp_size=kwargs.get("pcp_size", 1),
|
||||
dcp_size=kwargs.get("dcp_size", 1),
|
||||
data_parallel_index=0,
|
||||
data_parallel_size=1,
|
||||
data_parallel_rank_local=None,
|
||||
is_parallelism_agnostic=kwargs.get("is_parallelism_agnostic", False),
|
||||
),
|
||||
replicated_layout=kwargs.get("replicated_layout", False),
|
||||
|
||||
@@ -81,6 +81,8 @@ def _make_offloading_spec(
|
||||
pcp_size=1,
|
||||
dcp_size=1,
|
||||
data_parallel_index=0,
|
||||
data_parallel_size=1,
|
||||
data_parallel_rank_local=None,
|
||||
is_parallelism_agnostic=is_parallelism_agnostic,
|
||||
),
|
||||
replicated_layout=replicated_layout,
|
||||
|
||||
@@ -74,6 +74,8 @@ def _make_offloading_config(
|
||||
pcp_size=1,
|
||||
dcp_size=1,
|
||||
data_parallel_index=0,
|
||||
data_parallel_size=1,
|
||||
data_parallel_rank_local=None,
|
||||
is_parallelism_agnostic=is_parallelism_agnostic,
|
||||
),
|
||||
replicated_layout=replicated_layout,
|
||||
|
||||
@@ -213,6 +213,8 @@ def build_offloading_config(
|
||||
pcp_size=parallel_config.prefill_context_parallel_size,
|
||||
dcp_size=parallel_config.decode_context_parallel_size,
|
||||
data_parallel_index=parallel_config.data_parallel_index,
|
||||
data_parallel_size=parallel_config.data_parallel_size,
|
||||
data_parallel_rank_local=parallel_config.data_parallel_rank_local,
|
||||
is_parallelism_agnostic=is_parallelism_agnostic,
|
||||
),
|
||||
replicated_layout=replicated_layout,
|
||||
|
||||
@@ -48,6 +48,10 @@ class OffloadingParallelConfig:
|
||||
dcp_size: int
|
||||
# Data parallel replica index of this engine.
|
||||
data_parallel_index: int
|
||||
# Number of data parallel replicas.
|
||||
data_parallel_size: int
|
||||
# Local rank of the data parallel group, set only in SPMD mode.
|
||||
data_parallel_rank_local: int | None
|
||||
# True when the bytes that will be persisted for a block are portable
|
||||
# across parallelism configurations: for the direct layout, concatenating
|
||||
# the block's data across all workers in rank order yields the same bytes
|
||||
|
||||
Reference in New Issue
Block a user