diff --git a/.buildkite/intel_jobs/lm_eval_intel.yaml b/.buildkite/intel_jobs/lm_eval_intel.yaml index 966ced8156..11d3edc90e 100644 --- a/.buildkite/intel_jobs/lm_eval_intel.yaml +++ b/.buildkite/intel_jobs/lm_eval_intel.yaml @@ -23,6 +23,5 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install "gpt-oss[eval]==0.0.5" && - cd tests && + 'cd tests && pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt' diff --git a/.buildkite/intel_jobs/misc_intel.yaml b/.buildkite/intel_jobs/misc_intel.yaml index 3818339662..39b61b3f7d 100644 --- a/.buildkite/intel_jobs/misc_intel.yaml +++ b/.buildkite/intel_jobs/misc_intel.yaml @@ -69,8 +69,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install lm_eval[api]>=0.4.12 && - export VLLM_WORKER_MULTIPROC_METHOD=spawn && + 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && cd tests && pytest -v -s v1/logits_processors --ignore=v1/logits_processors/test_custom_online.py --ignore=v1/logits_processors/test_custom_offline.py && pytest -v -s v1/test_oracle.py v1/test_request.py v1/test_outputs.py && @@ -193,8 +192,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install modelscope\<1.38 && - cd tests && + 'cd tests && pytest -v -s test_regression.py' - label: Metrics, Tracing (2 GPUs) @@ -229,8 +227,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install opentelemetry-sdk\>=1.26.0 opentelemetry-api\>=1.26.0 opentelemetry-exporter-otlp\>=1.26.0 opentelemetry-semantic-conventions-ai\>=0.4.1 && - cd tests && + 'cd tests && pytest -v -s v1/tracing' - label: Async Engine, Inputs, Utils, Worker @@ -268,7 +265,6 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - pip install av && pytest -v -s detokenizer && pytest -v -s -m "not cpu_test" ./multimodal && pytest -v -s utils_ --ignore=utils_/test_mem_utils.py' diff --git a/.buildkite/intel_jobs/model_executor_intel.yaml b/.buildkite/intel_jobs/model_executor_intel.yaml index 14a853544e..d7c683f8ce 100644 --- a/.buildkite/intel_jobs/model_executor_intel.yaml +++ b/.buildkite/intel_jobs/model_executor_intel.yaml @@ -25,8 +25,6 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'apt-get update && apt-get install -y curl libsodium23 && - pip3 install tensorizer==2.10.1 && - pip3 install runai-model-streamer[s3,gcs,azure]\>=0.15.7 && export VLLM_WORKER_MULTIPROC_METHOD=spawn && export PYTHONFAULTHANDLER=1 && cd tests && diff --git a/.buildkite/intel_jobs/models_multimodal_intel.yaml b/.buildkite/intel_jobs/models_multimodal_intel.yaml index 44efa8e0f1..58ecb2d282 100644 --- a/.buildkite/intel_jobs/models_multimodal_intel.yaml +++ b/.buildkite/intel_jobs/models_multimodal_intel.yaml @@ -22,8 +22,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install av && - cd tests && + 'cd tests && pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" && pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model' @@ -96,8 +95,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install av && - cd tests && + 'cd tests && pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing' - label: Multi-Modal Processor # 44min @@ -121,9 +119,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install av matplotlib ftfy && - pip install open-clip-torch --no-deps && - cd tests && + 'cd tests && pytest -v -s models/multimodal/processing/test_tensor_schema.py --deselect="tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[deepseek-ai/DeepSeek-V4.1-Flash]" --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB' diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index 00f172ca88..8f4e83eee8 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -34,8 +34,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install tblib==3.1.0 && - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager && + 'python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN && @@ -129,8 +128,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'pip install av && - cd tests && + 'cd tests && pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py && pytest -v -s benchmarks/test_serve_cli.py' - label: "XPU quantization test" diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in index 6848dd98b7..e93b418e9a 100644 --- a/requirements/test/xpu.in +++ b/requirements/test/xpu.in @@ -1,7 +1,7 @@ -r ../common.txt # --- Test Infrastructure --- -tblib +tblib==3.1.0 pytest pytest_asyncio pytest-timeout @@ -17,9 +17,12 @@ accelerate arctic-inference lm_eval[api]>=0.4.12 modelscope<1.38 +runai-model-streamer[s3,gcs,azure]>=0.15.7 +tensorizer==2.10.1 transformers==5.16.1 # --- Audio Processing --- +av librosa audioread soxr @@ -29,7 +32,7 @@ soundfile # --- Tool Parsing & Evaluation --- blobfile rapidfuzz -gpt-oss +gpt-oss[eval]==0.0.5 schemathesis jiwer bm25s @@ -41,5 +44,18 @@ pqdm # --- Vision & Multimodal --- timm albumentations +ftfy +matplotlib decord==0.6.0 -mistral-common[image,audio] \ No newline at end of file +mistral-common[image,audio] + +open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py + +# --- Observability --- +opentelemetry-api>=1.26.0 +opentelemetry-exporter-otlp>=1.26.0 +opentelemetry-sdk>=1.26.0 +opentelemetry-semantic-conventions-ai>=0.4.1 + +cohere_melody>=0.11.1 # required for cohere command reasoning/tool parser tests +cohere>=7.0.0 # required for cohere chat v2 api tests (protocol/serving import cohere.types) diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 76453f50f2..6c8bc1fcdd 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -54,14 +54,34 @@ attrs==26.1.0 # referencing audioread==3.1.0 # via -r requirements/test/xpu.in +av==18.1.0 + # via -r requirements/test/xpu.in +azure-core==1.41.0 + # via + # azure-identity + # azure-storage-blob +azure-identity==1.25.3 + # via runai-model-streamer-azure +azure-storage-blob==12.30.1 + # via runai-model-streamer-azure blake3==1.0.9 # via -r requirements/test/../common.txt blobfile==3.2.0 - # via -r requirements/test/xpu.in + # via + # -r requirements/test/xpu.in + # gpt-oss bm25s==0.3.10 # via # -r requirements/test/xpu.in # mteb +boto3==1.43.87 + # via + # runai-model-streamer-s3 + # tensorizer +botocore==1.43.87 + # via + # boto3 + # s3transfer bounded-pool-executor==0.0.3 # via pqdm cachetools==7.1.7 @@ -94,16 +114,29 @@ click==8.4.2 # uvicorn cloudpickle==3.1.2 # via -r requirements/test/../common.txt +cohere==7.1.1 + # via -r requirements/test/xpu.in +cohere-melody==0.13.4 + # via -r requirements/test/xpu.in colorama==0.4.6 # via sacrebleu compressed-tensors==0.17.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt +contourpy==1.3.3 + # via matplotlib coverage==7.15.4 # via pytest-cov cryptography==50.0.0 - # via pyjwt + # via + # azure-identity + # azure-storage-blob + # google-auth + # msal + # pyjwt +cycler==0.12.1 + # via matplotlib dataproperty==1.1.1 # via # pytablewriter @@ -174,6 +207,8 @@ fastar==0.11.0 # via # fastapi # fastapi-cloud-cli +fastavro==1.12.2 + # via cohere filelock==3.32.3 # via # -c requirements/common.txt @@ -183,6 +218,8 @@ filelock==3.32.3 # huggingface-hub # modelscope # torch +fonttools==4.64.0 + # via matplotlib frozenlist==1.8.0 # via # aiohttp @@ -193,11 +230,36 @@ fsspec==2026.6.0 # evaluate # huggingface-hub # torch +ftfy==6.3.1 + # via + # -r requirements/test/xpu.in + # open-clip-torch +google-api-core==2.34.0 + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.57.0 + # via + # google-api-core + # google-cloud-core + # google-cloud-storage + # runai-model-streamer-gcs +google-cloud-core==2.7.0 + # via google-cloud-storage +google-cloud-storage==3.13.1 + # via runai-model-streamer-gcs +google-crc32c==1.8.0 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.10.2 + # via google-cloud-storage googleapis-common-protos==1.75.1 # via + # google-api-core # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -gpt-oss==0.0.9 +gpt-oss==0.0.5 # via -r requirements/test/xpu.in graphql-core==3.2.11 # via hypothesis-graphql @@ -212,6 +274,8 @@ harfile==0.5.0 # via schemathesis hf-xet==1.6.0 # via huggingface-hub +hiredis==3.4.1 + # via tensorizer html2text==2025.4.15 # via gpt-oss httpcore==1.0.9 @@ -223,6 +287,7 @@ httptools==0.8.0 httpx==0.28.1 # via # anthropic + # cohere # datasets # fastapi # fastapi-cloud-cli @@ -239,10 +304,13 @@ huggingface-hub==1.31.0 # accelerate # datasets # evaluate + # open-clip-torch # sentence-transformers # timm # tokenizers # transformers +humanize==4.16.0 + # via runai-model-streamer hypothesis==6.165.10 # via # hypothesis-graphql @@ -305,10 +373,13 @@ intel-sycl-rt==2026.0.0 # torch interegular==0.3.3 # via lm-format-enforcer +isodate==0.7.2 + # via azure-storage-blob jinja2==3.1.6 # via # -c requirements/xpu.txt # fastapi + # gpt-oss # lm-eval # torch jiter==0.16.0 @@ -318,7 +389,10 @@ jiter==0.16.0 jiwer==4.0.0 # via -r requirements/test/xpu.in jmespath==1.1.0 - # via model-hosting-container-standards + # via + # boto3 + # botocore + # model-hosting-container-standards joblib==1.5.3 # via # librosa @@ -335,16 +409,16 @@ jsonschema-rs==0.49.9 # via schemathesis jsonschema-specifications==2025.9.1 # via jsonschema -jupyter-client==8.9.1 - # via gpt-oss -jupyter-core==5.9.1 - # via jupyter-client +kiwisolver==1.5.1 + # via matplotlib lark==1.2.2 # via # -c requirements/common.txt # -r requirements/test/../common.txt lazy-loader==0.5 # via librosa +libnacl==2.1.0 + # via tensorizer librosa==1.0.0 # via -r requirements/test/xpu.in llguidance==1.7.6 @@ -372,6 +446,8 @@ markupsafe==3.0.3 # via # jinja2 # werkzeug +matplotlib==3.11.1 + # via -r requirements/test/xpu.in mbstrdecoder==1.1.5 # via # dataproperty @@ -408,6 +484,12 @@ more-itertools==11.1.0 # via lm-eval mpmath==1.3.0 # via sympy +msal==1.38.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.3.1 + # via azure-identity msgpack==1.2.1 # via librosa msgspec==0.21.1 @@ -436,18 +518,21 @@ numba==0.65.0 # via # -c requirements/xpu.txt # librosa -numpy==2.3.5 +numpy==2.2.6 # via # -r requirements/test/../common.txt # accelerate # albucore # albumentations # bm25s + # contourpy # datasets # decord # evaluate + # gpt-oss # librosa # lm-eval + # matplotlib # mistral-common # mteb # numba @@ -455,12 +540,14 @@ numpy==2.3.5 # pandas # pytrec-eval-terrier # rouge-score + # runai-model-streamer # sacrebleu # scikit-learn # scipy # sentence-transformers # soundfile # soxr + # tensorizer # torchvision # transformers # xgrammar @@ -487,10 +574,13 @@ onemkl-sycl-rng==2026.0.0 # via torch onemkl-sycl-sparse==2026.0.0 # via torch +open-clip-torch==2.32.0 + # via -r requirements/test/xpu.in openai==3.2.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # gpt-oss openai-harmony==0.0.8 # via # -c requirements/common.txt @@ -507,6 +597,7 @@ opentelemetry-api==1.44.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # -r requirements/test/xpu.in # mcp # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http @@ -516,6 +607,7 @@ opentelemetry-exporter-otlp==1.44.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # -r requirements/test/xpu.in opentelemetry-exporter-otlp-proto-common==1.44.0 # via # opentelemetry-exporter-otlp-proto-grpc @@ -533,6 +625,7 @@ opentelemetry-sdk==1.44.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # -r requirements/test/xpu.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions-ai @@ -544,6 +637,7 @@ opentelemetry-semantic-conventions-ai==0.5.1 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # -r requirements/test/xpu.in outlines-core==0.2.14 # via # -c requirements/common.txt @@ -557,6 +651,7 @@ packaging==26.3 # huggingface-hub # lazy-loader # lm-format-enforcer + # matplotlib # modelscope # pooch # pytest @@ -567,6 +662,7 @@ pandas==3.0.5 # via # datasets # evaluate + # gpt-oss partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt pathvalidate==3.3.1 @@ -574,12 +670,11 @@ pathvalidate==3.3.1 pillow==12.3.0 # via # -r requirements/test/../common.txt + # matplotlib # mistral-common # torchvision platformdirs==4.11.3 - # via - # jupyter-core - # pooch + # via pooch pluggy==1.6.0 # via # pytest @@ -609,20 +704,30 @@ propcache==0.5.2 # via # aiohttp # yarl +proto-plus==1.28.4 + # via google-api-core protobuf==7.35.1 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # google-api-core # googleapis-common-protos # opentelemetry-proto + # proto-plus + # tensorizer psutil==7.2.2 # via # -r requirements/test/../common.txt # accelerate + # tensorizer py-cpuinfo==9.0.0 # via -r requirements/test/../common.txt pyarrow==25.0.1 # via datasets +pyasn1==0.6.4 + # via pyasn1-modules +pyasn1-modules==0.4.2 + # via google-auth pybase64==1.5.0 # via -r requirements/test/../common.txt pycountry==26.2.16 @@ -637,6 +742,7 @@ pydantic==2.13.4 # -r requirements/test/../common.txt # albumentations # anthropic + # cohere # compressed-tensors # fastapi # fastapi-cloud-cli @@ -653,7 +759,9 @@ pydantic==2.13.4 # pydantic-settings # xgrammar pydantic-core==2.46.4 - # via pydantic + # via + # cohere + # pydantic pydantic-extra-types==2.11.1 # via # fastapi @@ -667,7 +775,11 @@ pygments==2.21.0 # pytest # rich pyjwt==2.13.0 - # via mcp + # via + # mcp + # msal +pyparsing==3.3.2 + # via matplotlib pyrate-limiter==4.4.0 # via schemathesis pystemmer==3.1.0 @@ -700,7 +812,8 @@ pytest-timeout==2.4.0 # via -r requirements/test/xpu.in python-dateutil==2.9.0.post0 # via - # jupyter-client + # botocore + # matplotlib # pandas # typepy python-dotenv==1.2.3 @@ -735,11 +848,12 @@ pyzmq==27.1.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt - # jupyter-client rapidfuzz==3.14.5 # via # -r requirements/test/xpu.in # jiwer +redis==8.1.0 + # via tensorizer referencing==0.37.0 # via # jsonschema @@ -748,6 +862,7 @@ regex==2026.7.19 # via # -r requirements/test/../common.txt # nltk + # open-clip-torch # sacrebleu # tiktoken # transformers @@ -755,13 +870,18 @@ requests==2.34.2 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # azure-core + # cohere # datasets # docker # evaluate + # google-api-core + # google-cloud-storage # gpt-oss # lm-eval # mistral-common # modelscope + # msal # mteb # opentelemetry-exporter-otlp-proto-http # pooch @@ -786,6 +906,16 @@ rpds-py==2026.6.3 # via # jsonschema # referencing +runai-model-streamer==0.16.1 + # via -r requirements/test/xpu.in +runai-model-streamer-azure==0.16.1 + # via runai-model-streamer +runai-model-streamer-gcs==0.16.1 + # via runai-model-streamer +runai-model-streamer-s3==0.16.1 + # via runai-model-streamer +s3transfer==0.19.2 + # via boto3 sacrebleu==2.6.0 # via lm-eval safetensors==0.8.0 @@ -793,6 +923,7 @@ safetensors==0.8.0 # -c requirements/common.txt # -r requirements/test/../common.txt # accelerate + # open-clip-torch # timm # transformers schemathesis==4.24.3 @@ -887,7 +1018,7 @@ tbb==2023.0.0 # intel-opencl-rt # mkl # torch -tblib==3.2.2 +tblib==3.1.0 # via -r requirements/test/xpu.in tcmlib==1.5.0 # via @@ -900,6 +1031,8 @@ tenacity==9.1.4 # via # gpt-oss # lm-eval +tensorizer==2.10.1 + # via -r requirements/test/xpu.in termcolor==3.3.0 # via gpt-oss threadpoolctl==3.6.0 @@ -912,11 +1045,14 @@ tiktoken==0.14.0 # lm-eval # mistral-common timm==1.0.28 - # via -r requirements/test/xpu.in + # via + # -r requirements/test/xpu.in + # open-clip-torch tokenizers==0.23.1 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # cohere # sentence-transformers # transformers torch==2.13.0+xpu @@ -925,32 +1061,33 @@ torch==2.13.0+xpu # accelerate # compressed-tensors # mteb + # open-clip-torch + # runai-model-streamer # sentence-transformers + # tensorizer # timm # torchvision # xgrammar torchvision==0.28.0+xpu - # via timm -tornado==6.5.8 - # via jupyter-client + # via + # open-clip-torch + # timm tqdm==4.70.0 # via # -r requirements/test/../common.txt # datasets # evaluate + # gpt-oss # huggingface-hub # lm-eval # modelscope # mteb # nltk + # open-clip-torch # openai # pqdm # sentence-transformers # transformers -traitlets==5.16.1 - # via - # jupyter-client - # jupyter-core transformers==5.16.1 # via # -c requirements/common.txt @@ -982,6 +1119,8 @@ typer==0.27.1 # fastapi-cli # fastapi-cloud-cli # transformers +types-requests==2.33.0.20260712 + # via cohere typing-extensions==4.16.0 # via # -c requirements/common.txt @@ -991,12 +1130,15 @@ typing-extensions==4.16.0 # anthropic # anyio # apache-tvm-ffi + # azure-core + # azure-identity + # azure-storage-blob # chz + # cohere # fastapi # grpcio # httpx2 # huggingface-hub - # jupyter-client # lm-eval # mcp # mcp-types @@ -1035,10 +1177,12 @@ umf==1.1.0 urllib3==2.7.0 # via # blobfile + # botocore # docker # modelscope # requests # sentry-sdk + # types-requests uvicorn==0.52.3 # via # fastapi @@ -1052,6 +1196,8 @@ watchfiles==1.2.0 # via # -r requirements/test/../common.txt # uvicorn +wcwidth==0.8.3 + # via ftfy websockets==17.0.1 # via uvicorn werkzeug==3.1.8