From 5cc4876630659b88c04c444a279556081a6ee82a Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Thu, 1 Jan 2026 21:29:42 -0600 Subject: [PATCH] [ROCm][CI] Fix failure in Language Models Tests (Extra Standard) by reducing agent pool size (#31553) Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 0eb64e566e..ea117d4ed7 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -859,7 +859,7 @@ steps: - label: Language Models Tests (Extra Standard) %N timeout_in_minutes: 45 mirror_hardwares: [amdexperimental] - agent_pool: mi325_8 + agent_pool: mi325_2 # grade: Blocking torch_nightly: true source_file_dependencies: @@ -871,6 +871,7 @@ steps: # Shard slow subset of standard language models tests. Only run when model # source is modified, or when specified test files are modified - pip freeze | grep -E 'torch' + - export TORCH_NCCL_BLOCKING_WAIT=1 - pytest -v -s models/language -m 'core_model and slow_test' \ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT \ --shard-id=$$BUILDKITE_PARALLEL_JOB