[UX] DCP Topology Validation (#49777)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
+23
-17
@@ -1298,27 +1298,33 @@ class ModelConfig:
|
|||||||
decode_context_parallel_size = parallel_config.decode_context_parallel_size
|
decode_context_parallel_size = parallel_config.decode_context_parallel_size
|
||||||
if decode_context_parallel_size > 1 and not self.use_mla:
|
if decode_context_parallel_size > 1 and not self.use_mla:
|
||||||
total_num_kv_heads = self.get_total_num_kv_heads()
|
total_num_kv_heads = self.get_total_num_kv_heads()
|
||||||
assert tensor_parallel_size > total_num_kv_heads, (
|
if tensor_parallel_size <= total_num_kv_heads:
|
||||||
f"tensor parallel size {tensor_parallel_size} must be greater "
|
raise ValueError(
|
||||||
f"than total num kv heads {total_num_kv_heads} when enable "
|
"Decode context parallelism for GQA/MQA requires "
|
||||||
f"decode context parallel for GQA/MQA"
|
f"`--tensor-parallel-size` ({tensor_parallel_size}) to be "
|
||||||
)
|
"greater than the model's total number of KV heads "
|
||||||
|
f"({total_num_kv_heads}). Increase `--tensor-parallel-size` "
|
||||||
|
"or set `--decode-context-parallel-size 1`."
|
||||||
|
)
|
||||||
|
|
||||||
max_dcp_size = tensor_parallel_size // total_num_kv_heads
|
max_dcp_size = tensor_parallel_size // total_num_kv_heads
|
||||||
assert decode_context_parallel_size <= max_dcp_size, (
|
if decode_context_parallel_size > max_dcp_size:
|
||||||
f"decode context parallel size must less than or equal to "
|
raise ValueError(
|
||||||
f"(tensor parallel size {tensor_parallel_size} // total "
|
"`--decode-context-parallel-size` "
|
||||||
f"num kv heads {total_num_kv_heads}) = {max_dcp_size}, "
|
f"({decode_context_parallel_size}) exceeds the maximum "
|
||||||
f"but got {decode_context_parallel_size}"
|
f"supported value ({max_dcp_size}) for "
|
||||||
)
|
f"`--tensor-parallel-size` ({tensor_parallel_size}) and "
|
||||||
|
f"{total_num_kv_heads} model KV heads."
|
||||||
|
)
|
||||||
|
|
||||||
num_q_per_kv = total_num_attention_heads // total_num_kv_heads
|
num_q_per_kv = total_num_attention_heads // total_num_kv_heads
|
||||||
assert num_q_per_kv % decode_context_parallel_size == 0, (
|
if num_q_per_kv % decode_context_parallel_size != 0:
|
||||||
f"Total number of q per kv attn heads ({num_q_per_kv})"
|
raise ValueError(
|
||||||
" must be divisible by dcp world size when enable "
|
"The model's number of query heads per KV head "
|
||||||
"decode context parallel for GQA "
|
f"({num_q_per_kv}) must be divisible by "
|
||||||
f"({parallel_config.decode_context_parallel_size})."
|
"`--decode-context-parallel-size` "
|
||||||
)
|
f"({decode_context_parallel_size}) for GQA/MQA."
|
||||||
|
)
|
||||||
|
|
||||||
# torch_shm uses a single IPC queue to rank 0; DP>1 is
|
# torch_shm uses a single IPC queue to rank 0; DP>1 is
|
||||||
# incompatible because API servers can't know which
|
# incompatible because API servers can't know which
|
||||||
|
|||||||
Reference in New Issue
Block a user