[CI/Build][The Rock][BugFix] Use fork method in test_multiproc_executor_multi_node for py 3.14 compat and fix test_multiproc_executor_shutdown_cleanup (#48655)
Signed-off-by: Randall Smith <Randall.Smith@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
1dca300653
commit
60d443f738
@@ -254,8 +254,8 @@ def test_multiproc_executor_shutdown_cleanup():
|
||||
for worker in executor.workers:
|
||||
assert not worker.proc.is_alive(), "Worker processes should be terminated"
|
||||
|
||||
# Verify shutdown event is set
|
||||
assert executor.shutdown_event.is_set(), "Shutdown event should be set"
|
||||
# Verify shutdown flag is set
|
||||
assert executor.shutting_down, "Shutdown flag should be set"
|
||||
|
||||
# Multiple shutdowns should be safe (idempotent)
|
||||
executor.shutdown()
|
||||
@@ -292,7 +292,6 @@ def test_multiproc_executor_pipeline_parallel():
|
||||
"Max concurrent batches should follow the configured PP/async "
|
||||
"scheduling policy"
|
||||
)
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
executor.shutdown()
|
||||
@@ -338,6 +337,10 @@ def test_multiproc_executor_multi_node():
|
||||
- Node 1 (rank 1): Uses GPUs 2,3 (CUDA_VISIBLE_DEVICES=2,3) with TP=2
|
||||
Total world_size = 4, nnodes = 2
|
||||
"""
|
||||
# Python 3.14+ changed default multiprocessing start method to 'forkserver'
|
||||
# which cannot pickle nested functions. Use 'fork' for this test.
|
||||
mp_ctx = multiprocessing.get_context("fork")
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("", 0))
|
||||
port = s.getsockname()[1]
|
||||
@@ -405,12 +408,12 @@ def test_multiproc_executor_multi_node():
|
||||
executor.shutdown()
|
||||
|
||||
# Create a queue to collect results from both processes
|
||||
result_queue: multiprocessing.Queue[dict[str, int | bool]] = multiprocessing.Queue()
|
||||
result_queue: multiprocessing.Queue[dict[str, int | bool]] = mp_ctx.Queue()
|
||||
|
||||
# Start both node processes
|
||||
processes = []
|
||||
for node_rank in range(2):
|
||||
p = multiprocessing.Process(
|
||||
p = mp_ctx.Process(
|
||||
target=run_node,
|
||||
args=(node_rank, result_queue, port),
|
||||
name=f"Node{node_rank}",
|
||||
|
||||
@@ -246,6 +246,17 @@ class MultiprocExecutor(Executor):
|
||||
|
||||
self.output_rank = self._get_output_rank()
|
||||
|
||||
def get_response_mqs(self, unique_reply_rank: int = -1) -> list[MessageQueue]:
|
||||
assert unique_reply_rank >= -1 and unique_reply_rank < self.world_size, (
|
||||
f"unique_reply_rank must be -1 or < world_size,"
|
||||
f"unique_reply_rank = {unique_reply_rank}, "
|
||||
f"world_size={self.world_size}"
|
||||
)
|
||||
ranks = (
|
||||
[unique_reply_rank] if unique_reply_rank != -1 else range(self.world_size)
|
||||
)
|
||||
return [self.workers[rank].worker_response_mq for rank in ranks]
|
||||
|
||||
def _get_parallel_sizes(self) -> tuple[int, int, int]:
|
||||
self.world_size = self.parallel_config.world_size
|
||||
assert self.world_size % self.parallel_config.nnodes_within_dp == 0, (
|
||||
|
||||
Reference in New Issue
Block a user