[UX] DCP Topology Validation (#49777)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
+23
-17
@@ -1298,27 +1298,33 @@ class ModelConfig:
|
||||
decode_context_parallel_size = parallel_config.decode_context_parallel_size
|
||||
if decode_context_parallel_size > 1 and not self.use_mla:
|
||||
total_num_kv_heads = self.get_total_num_kv_heads()
|
||||
assert tensor_parallel_size > total_num_kv_heads, (
|
||||
f"tensor parallel size {tensor_parallel_size} must be greater "
|
||||
f"than total num kv heads {total_num_kv_heads} when enable "
|
||||
f"decode context parallel for GQA/MQA"
|
||||
)
|
||||
if tensor_parallel_size <= total_num_kv_heads:
|
||||
raise ValueError(
|
||||
"Decode context parallelism for GQA/MQA requires "
|
||||
f"`--tensor-parallel-size` ({tensor_parallel_size}) to be "
|
||||
"greater than the model's total number of KV heads "
|
||||
f"({total_num_kv_heads}). Increase `--tensor-parallel-size` "
|
||||
"or set `--decode-context-parallel-size 1`."
|
||||
)
|
||||
|
||||
max_dcp_size = tensor_parallel_size // total_num_kv_heads
|
||||
assert decode_context_parallel_size <= max_dcp_size, (
|
||||
f"decode context parallel size must less than or equal to "
|
||||
f"(tensor parallel size {tensor_parallel_size} // total "
|
||||
f"num kv heads {total_num_kv_heads}) = {max_dcp_size}, "
|
||||
f"but got {decode_context_parallel_size}"
|
||||
)
|
||||
if decode_context_parallel_size > max_dcp_size:
|
||||
raise ValueError(
|
||||
"`--decode-context-parallel-size` "
|
||||
f"({decode_context_parallel_size}) exceeds the maximum "
|
||||
f"supported value ({max_dcp_size}) for "
|
||||
f"`--tensor-parallel-size` ({tensor_parallel_size}) and "
|
||||
f"{total_num_kv_heads} model KV heads."
|
||||
)
|
||||
|
||||
num_q_per_kv = total_num_attention_heads // total_num_kv_heads
|
||||
assert num_q_per_kv % decode_context_parallel_size == 0, (
|
||||
f"Total number of q per kv attn heads ({num_q_per_kv})"
|
||||
" must be divisible by dcp world size when enable "
|
||||
"decode context parallel for GQA "
|
||||
f"({parallel_config.decode_context_parallel_size})."
|
||||
)
|
||||
if num_q_per_kv % decode_context_parallel_size != 0:
|
||||
raise ValueError(
|
||||
"The model's number of query heads per KV head "
|
||||
f"({num_q_per_kv}) must be divisible by "
|
||||
"`--decode-context-parallel-size` "
|
||||
f"({decode_context_parallel_size}) for GQA/MQA."
|
||||
)
|
||||
|
||||
# torch_shm uses a single IPC queue to rank 0; DP>1 is
|
||||
# incompatible because API servers can't know which
|
||||
|
||||
Reference in New Issue
Block a user