From 1d694e78c954027a42c909cd3488274e4c5ccd9c Mon Sep 17 00:00:00 2001 From: "wang.yuqi" Date: Fri, 8 May 2026 10:42:12 +0800 Subject: [PATCH] [Examples][last/6] Resettle examples. (#41084) Signed-off-by: wang.yuqi Signed-off-by: wang.yuqi Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../scripts/hardware_ci/run-tpu-v1-test.sh | 2 -- .buildkite/test-amd.yaml | 12 ++++----- .buildkite/test_areas/misc.yaml | 4 +-- .buildkite/test_areas/model_runner_v2.yaml | 6 ++--- docker/Dockerfile | 2 +- docs/contributing/model/transcription.md | 2 +- docs/deployment/frameworks/helm.md | 2 +- docs/deployment/frameworks/lws.md | 4 +-- .../retrieval_augmented_generation.md | 4 +-- docs/deployment/frameworks/skypilot.md | 4 +-- docs/deployment/frameworks/streamlit.md | 2 +- docs/deployment/integrations/kthena.md | 8 +++--- docs/examples/README.md | 6 ++--- docs/features/quantization/auto_awq.md | 2 +- docs/features/quantization/gptqmodel.md | 2 +- docs/features/reasoning_outputs.md | 2 +- .../installation/gpu.xpu.inc.md | 2 +- docs/models/extensions/tensorizer.md | 8 +++--- docs/serving/openai_compatible_server.md | 6 ++--- .../chatbot}/api_client.py | 0 .../gradio_openai_chatbot_webserver.py | 2 +- .../chatbot}/gradio_webserver.py | 2 +- .../streamlit_openai_chatbot_webserver.py | 0 ...val_augmented_generation_with_langchain.py | 0 ...al_augmented_generation_with_llamaindex.py | 0 .../async_llm_streaming.py | 2 +- .../chart-helm/.helmignore | 0 .../chart-helm/Chart.yaml | 0 .../chart-helm/README.md | 0 .../chart-helm/ct.yaml | 0 .../chart-helm/lintconf.yaml | 0 .../chart-helm/templates/_helpers.tpl | 0 .../chart-helm/templates/configmap.yaml | 0 .../chart-helm/templates/custom-objects.yaml | 0 .../chart-helm/templates/deployment.yaml | 0 .../chart-helm/templates/hpa.yaml | 0 .../chart-helm/templates/job.yaml | 0 .../templates/poddisruptionbudget.yaml | 0 .../chart-helm/templates/pvc.yaml | 0 .../chart-helm/templates/secrets.yaml | 0 .../chart-helm/templates/service.yaml | 0 .../chart-helm/tests/deployment_test.yaml | 0 .../chart-helm/tests/job_test.yaml | 0 .../chart-helm/tests/pvc_test.yaml | 0 .../chart-helm/values.schema.json | 0 .../chart-helm/values.yaml | 0 .../llm_engine_example.py | 0 .../sagemaker-entrypoint.sh | 0 .../disagg_proxy_multiturn.py | 0 .../logging_configuration.md | 0 .../tensorize_vllm_model.py | 10 +++---- examples/observability/dashboards/README.md | 2 +- examples/online_serving/utils.py | 26 ------------------- .../openai_embedding_long_text/service.sh | 2 +- .../speech_to_text/lid/openai_lid_client.py | 6 ++--- requirements/common.txt | 2 +- tests/lora/test_llama_tp.py | 2 +- .../tensorizer_loader/test_tensorizer.py | 2 +- vllm/assets/video.py | 2 +- .../model_executor/model_loader/tensorizer.py | 4 +-- .../model_loader/tensorizer_loader.py | 4 +-- 61 files changed, 60 insertions(+), 88 deletions(-) rename examples/{online_serving => applications/chatbot}/api_client.py (100%) rename examples/{online_serving => applications/chatbot}/gradio_openai_chatbot_webserver.py (97%) rename examples/{online_serving => applications/chatbot}/gradio_webserver.py (97%) rename examples/{online_serving => applications/chatbot}/streamlit_openai_chatbot_webserver.py (100%) rename examples/{online_serving => applications/rag}/retrieval_augmented_generation_with_langchain.py (100%) rename examples/{online_serving => applications/rag}/retrieval_augmented_generation_with_llamaindex.py (100%) rename examples/{offline_inference => deployment}/async_llm_streaming.py (98%) rename examples/{online_serving => deployment}/chart-helm/.helmignore (100%) rename examples/{online_serving => deployment}/chart-helm/Chart.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/README.md (100%) rename examples/{online_serving => deployment}/chart-helm/ct.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/lintconf.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/templates/_helpers.tpl (100%) rename examples/{online_serving => deployment}/chart-helm/templates/configmap.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/templates/custom-objects.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/templates/deployment.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/templates/hpa.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/templates/job.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/templates/poddisruptionbudget.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/templates/pvc.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/templates/secrets.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/templates/service.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/tests/deployment_test.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/tests/job_test.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/tests/pvc_test.yaml (100%) rename examples/{online_serving => deployment}/chart-helm/values.schema.json (100%) rename examples/{online_serving => deployment}/chart-helm/values.yaml (100%) rename examples/{offline_inference => deployment}/llm_engine_example.py (100%) rename examples/{online_serving => deployment}/sagemaker-entrypoint.sh (100%) rename examples/{online_serving => disaggregated}/disaggregated_serving/disagg_proxy_multiturn.py (100%) rename examples/{others => features}/logging_configuration.md (100%) rename examples/{others => features}/tensorize_vllm_model.py (97%) delete mode 100644 examples/online_serving/utils.py diff --git a/.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh b/.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh index feaf2b3562..61ebddf82e 100755 --- a/.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh +++ b/.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh @@ -136,8 +136,6 @@ run_and_track_test 3 "test_accuracy.py::test_lm_eval_accuracy_v1_engine" \ "python3 -m pytest -s -v /workspace/vllm/tests/entrypoints/llm/test_accuracy.py::test_lm_eval_accuracy_v1_engine" run_and_track_test 4 "test_quantization_accuracy.py" \ "python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_quantization_accuracy.py" -run_and_track_test 5 "examples/offline_inference/tpu.py" \ - "python3 /workspace/vllm/examples/offline_inference/tpu.py" run_and_track_test 6 "test_tpu_model_runner.py" \ "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/worker/test_tpu_model_runner.py" run_and_track_test 7 "test_sampler.py" \ diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index b3c77dcac7..02820250f9 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -394,8 +394,8 @@ steps: - python3 pooling/embed/vision_embedding_offline.py --seed 0 # Features demo - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 offline_inference/llm_engine_example.py - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors + - python3 deployment/llm_engine_example.py + - python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 @@ -1649,8 +1649,8 @@ steps: - python3 pooling/embed/vision_embedding_offline.py --seed 0 # Features demo - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 offline_inference/llm_engine_example.py - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors + - python3 deployment/llm_engine_example.py + - python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 @@ -2930,8 +2930,8 @@ steps: - python3 pooling/embed/vision_embedding_offline.py --seed 0 # Features demo - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 offline_inference/llm_engine_example.py - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors + - python3 deployment/llm_engine_example.py + - python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index c34d4c10b4..a9ae038b7b 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -117,8 +117,8 @@ steps: - python3 pooling/embed/vision_embedding_offline.py --seed 0 # for features demo - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 offline_inference/llm_engine_example.py - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors + - python3 deployment/llm_engine_example.py + - python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 diff --git a/.buildkite/test_areas/model_runner_v2.yaml b/.buildkite/test_areas/model_runner_v2.yaml index 6a4338a5e4..93b5197a6f 100644 --- a/.buildkite/test_areas/model_runner_v2.yaml +++ b/.buildkite/test_areas/model_runner_v2.yaml @@ -37,7 +37,7 @@ steps: - examples/generate/multimodal/ - examples/features/ - examples/pooling/embed/vision_embedding_offline.py - - examples/others/tensorize_vllm_model.py + - examples/features/tensorize_vllm_model.py commands: - set -x - export VLLM_USE_V2_MODEL_RUNNER=1 @@ -55,8 +55,8 @@ steps: - python3 pooling/embed/vision_embedding_offline.py --seed 0 # for features demo - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 offline_inference/llm_engine_example.py - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors + - python3 deployment/llm_engine_example.py + - python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 diff --git a/docker/Dockerfile b/docker/Dockerfile index fd0622e241..1051ca97d3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -860,7 +860,7 @@ LABEL org.opencontainers.image.source="https://github.com/vllm-project/vllm" \ # define sagemaker first, so it is not default from `docker build` FROM vllm-openai-base AS vllm-sagemaker -COPY examples/online_serving/sagemaker-entrypoint.sh . +COPY examples/deployment/sagemaker-entrypoint.sh . RUN chmod +x sagemaker-entrypoint.sh ENTRYPOINT ["./sagemaker-entrypoint.sh"] diff --git a/docs/contributing/model/transcription.md b/docs/contributing/model/transcription.md index 3e2ee38d2b..b076ef84a4 100644 --- a/docs/contributing/model/transcription.md +++ b/docs/contributing/model/transcription.md @@ -278,7 +278,7 @@ Once your model implements `SupportsTranscription`, you can test the endpoints ( http://localhost:8000/v1/audio/translations ``` -Or check out more examples in [examples/online_serving](../../../examples/online_serving). +Or check out more examples in [examples/speech_to_text](../../../examples/speech_to_text). !!! note - If your model handles chunking internally (e.g., via its processor or encoder), set `min_energy_split_window_size=None` in the returned `SpeechToTextConfig` to disable server-side chunking. diff --git a/docs/deployment/frameworks/helm.md b/docs/deployment/frameworks/helm.md index a0aee70b1b..45924dd6f7 100644 --- a/docs/deployment/frameworks/helm.md +++ b/docs/deployment/frameworks/helm.md @@ -17,7 +17,7 @@ Before you begin, ensure that you have the following: ## Installing the chart -This guide uses the Helm chart at [examples/online_serving/chart-helm](../../../examples/online_serving/chart-helm). +This guide uses the Helm chart at [examples/deployment/chart-helm](../../../examples/deployment/chart-helm). To install the chart with the release name `test-vllm`: diff --git a/docs/deployment/frameworks/lws.md b/docs/deployment/frameworks/lws.md index 14710a8dc3..47586bcd70 100644 --- a/docs/deployment/frameworks/lws.md +++ b/docs/deployment/frameworks/lws.md @@ -40,7 +40,7 @@ Deploy the following yaml file `lws.yaml` command: - sh - -c - - "bash /vllm-workspace/examples/online_serving/multi-node-serving.sh leader --ray_cluster_size=$(LWS_GROUP_SIZE); + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=$(LWS_GROUP_SIZE); vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline_parallel_size 2" resources: limits: @@ -73,7 +73,7 @@ Deploy the following yaml file `lws.yaml` command: - sh - -c - - "bash /vllm-workspace/examples/online_serving/multi-node-serving.sh worker --ray_address=$(LWS_LEADER_ADDRESS)" + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(LWS_LEADER_ADDRESS)" resources: limits: nvidia.com/gpu: "8" diff --git a/docs/deployment/frameworks/retrieval_augmented_generation.md b/docs/deployment/frameworks/retrieval_augmented_generation.md index 8a5d18807d..c23862cde7 100644 --- a/docs/deployment/frameworks/retrieval_augmented_generation.md +++ b/docs/deployment/frameworks/retrieval_augmented_generation.md @@ -36,7 +36,7 @@ pip install -U vllm \ vllm serve qwen/Qwen1.5-0.5B-Chat --port 8001 ``` -1. Use the script: [examples/online_serving/retrieval_augmented_generation_with_langchain.py](../../../examples/online_serving/retrieval_augmented_generation_with_langchain.py) +1. Use the script: [examples/applications/rag/retrieval_augmented_generation_with_langchain.py](../../../examples/applications/rag/retrieval_augmented_generation_with_langchain.py) 1. Run the script @@ -74,7 +74,7 @@ pip install vllm \ vllm serve qwen/Qwen1.5-0.5B-Chat --port 8001 ``` -1. Use the script: [examples/online_serving/retrieval_augmented_generation_with_llamaindex.py](../../../examples/online_serving/retrieval_augmented_generation_with_llamaindex.py) +1. Use the script: [examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py](../../../examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py) 1. Run the script: diff --git a/docs/deployment/frameworks/skypilot.md b/docs/deployment/frameworks/skypilot.md index e9b0d5f067..e032769f15 100644 --- a/docs/deployment/frameworks/skypilot.md +++ b/docs/deployment/frameworks/skypilot.md @@ -59,7 +59,7 @@ See the vLLM SkyPilot YAML for serving, [serving.yaml](https://github.com/skypil echo 'Starting gradio server...' git clone https://github.com/vllm-project/vllm.git || true - python vllm/examples/online_serving/gradio_openai_chatbot_webserver.py \ + python vllm/examples/applications/chatbot/gradio_openai_chatbot_webserver.py \ -m $MODEL_NAME \ --port 8811 \ --model-url http://localhost:8081/v1 \ @@ -305,7 +305,7 @@ It is also possible to access the Llama-3 service with a separate GUI frontend, echo 'Starting gradio server...' git clone https://github.com/vllm-project/vllm.git || true - python vllm/examples/online_serving/gradio_openai_chatbot_webserver.py \ + python vllm/examples/applications/api_client/gradio_openai_chatbot_webserver.py \ -m $MODEL_NAME \ --port 8811 \ --model-url http://$ENDPOINT/v1 \ diff --git a/docs/deployment/frameworks/streamlit.md b/docs/deployment/frameworks/streamlit.md index 1b214e1a32..6c0c8c6c14 100644 --- a/docs/deployment/frameworks/streamlit.md +++ b/docs/deployment/frameworks/streamlit.md @@ -20,7 +20,7 @@ pip install vllm streamlit openai vllm serve Qwen/Qwen1.5-0.5B-Chat ``` -1. Use the script: [examples/online_serving/streamlit_openai_chatbot_webserver.py](../../../examples/online_serving/streamlit_openai_chatbot_webserver.py) +1. Use the script: [examples/applications/chatbot/streamlit_openai_chatbot_webserver.py](../../../examples/applications/chatbot/streamlit_openai_chatbot_webserver.py) 1. Start the streamlit web UI and start to chat: diff --git a/docs/deployment/integrations/kthena.md b/docs/deployment/integrations/kthena.md index 483dd74744..0989e5d67f 100644 --- a/docs/deployment/integrations/kthena.md +++ b/docs/deployment/integrations/kthena.md @@ -78,7 +78,7 @@ Key points from the example YAML: - sh - -c - > - bash /vllm-workspace/examples/online_serving/multi-node-serving.sh leader --ray_cluster_size=2; + bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=2; python3 -m vllm.entrypoints.openai.api_server --port 8080 --model meta-llama/Llama-3.1-405B-Instruct @@ -93,7 +93,7 @@ Key points from the example YAML: - sh - -c - > - bash /vllm-workspace/examples/online_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS) + bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS) ``` --- @@ -144,7 +144,7 @@ spec: command: - sh - -c - - "bash /vllm-workspace/examples/online_serving/multi-node-serving.sh leader --ray_cluster_size=2; + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=2; python3 -m vllm.entrypoints.openai.api_server --port 8080 --model meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size 2" resources: limits: @@ -178,7 +178,7 @@ spec: command: - sh - -c - - "bash /vllm-workspace/examples/online_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS)" + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS)" resources: limits: nvidia.com/gpu: "8" diff --git a/docs/examples/README.md b/docs/examples/README.md index f5707ab6ee..e4cd1c87f4 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -2,6 +2,6 @@ vLLM's examples are split into three categories: -- If you are using vLLM from within Python code, see the [Offline Inference](../../examples/offline_inference) section. -- If you are using vLLM from an HTTP application or client, see the [Online Serving](../../examples/online_serving) section. -- For examples of using some of vLLM's advanced features (e.g. LMCache or Tensorizer) which are not specific to either of the above use cases, see the [Others](../../examples/others) section. +- If you are using vLLM from within Python code, see the [Offline Inference](.) section. +- If you are using vLLM from an HTTP application or client, see the [Online Serving](.) section. +- For examples of using some of vLLM's advanced features (e.g. LMCache or Tensorizer) which are not specific to either of the above use cases, see the [Others](.) section. diff --git a/docs/features/quantization/auto_awq.md b/docs/features/quantization/auto_awq.md index e77e8b5a1f..e93005f263 100644 --- a/docs/features/quantization/auto_awq.md +++ b/docs/features/quantization/auto_awq.md @@ -47,7 +47,7 @@ After installing AutoAWQ, you are ready to quantize a model. Please refer to the To run an AWQ model with vLLM, you can use [TheBloke/Llama-2-7b-Chat-AWQ](https://huggingface.co/TheBloke/Llama-2-7b-Chat-AWQ) with the following command: ```bash -python examples/offline_inference/llm_engine_example.py \ +python examples/deployment/llm_engine_example.py \ --model TheBloke/Llama-2-7b-Chat-AWQ \ --quantization awq ``` diff --git a/docs/features/quantization/gptqmodel.md b/docs/features/quantization/gptqmodel.md index f14a931725..636a952b65 100644 --- a/docs/features/quantization/gptqmodel.md +++ b/docs/features/quantization/gptqmodel.md @@ -58,7 +58,7 @@ Here is an example of how to quantize `meta-llama/Llama-3.2-1B-Instruct`: To run an GPTQModel quantized model with vLLM, you can use [DeepSeek-R1-Distill-Qwen-7B-gptqmodel-4bit-vortex-v2](https://huggingface.co/ModelCloud/DeepSeek-R1-Distill-Qwen-7B-gptqmodel-4bit-vortex-v2) with the following command: ```bash -python examples/offline_inference/llm_engine_example.py \ +python examples/deployment/llm_engine_example.py \ --model ModelCloud/DeepSeek-R1-Distill-Qwen-7B-gptqmodel-4bit-vortex-v2 ``` diff --git a/docs/features/reasoning_outputs.md b/docs/features/reasoning_outputs.md index 374149786e..f1cc18a25c 100644 --- a/docs/features/reasoning_outputs.md +++ b/docs/features/reasoning_outputs.md @@ -157,7 +157,7 @@ OpenAI Python client library does not officially support `reasoning` attribute f print(content, end="", flush=True) ``` -Remember to check whether the `reasoning` exists in the response before accessing it. You could check out the [example](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/openai_chat_completion_with_reasoning_streaming.py). +Remember to check whether the `reasoning` exists in the response before accessing it. You could check out the [example](https://github.com/vllm-project/vllm/blob/main/examples/reasoning/openai_chat_completion_with_reasoning_streaming.py). ## Tool Calling diff --git a/docs/getting_started/installation/gpu.xpu.inc.md b/docs/getting_started/installation/gpu.xpu.inc.md index 8c28258228..e8b74a06f0 100644 --- a/docs/getting_started/installation/gpu.xpu.inc.md +++ b/docs/getting_started/installation/gpu.xpu.inc.md @@ -88,7 +88,7 @@ vllm serve facebook/opt-13b \ -tp=8 ``` -By default, a ray instance will be launched automatically if no existing one is detected in the system, with `num-gpus` equals to `parallel_config.world_size`. We recommend properly starting a ray cluster before execution, referring to the [examples/online_serving/run_cluster.sh](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/run_cluster.sh) helper script. +By default, a ray instance will be launched automatically if no existing one is detected in the system, with `num-gpus` equals to `parallel_config.world_size`. We recommend properly starting a ray cluster before execution, referring to the [examples/ray_serving/run_cluster.sh](https://github.com/vllm-project/vllm/blob/main/examples/ray_serving/run_cluster.sh) helper script. --8<-- [end:supported-features] --8<-- [start:distributed-backend] diff --git a/docs/models/extensions/tensorizer.md b/docs/models/extensions/tensorizer.md index 3df80d5af6..0f3ee1ede4 100644 --- a/docs/models/extensions/tensorizer.md +++ b/docs/models/extensions/tensorizer.md @@ -14,7 +14,7 @@ To install `tensorizer`, run `pip install vllm[tensorizer]`. ## The basics To load a model using Tensorizer, the model first needs to be serialized by -Tensorizer. [The example script](../../examples/others/tensorize_vllm_model.md) takes care of this process. +Tensorizer. [The example script](../../../examples/features/tensorize_vllm_model.py) takes care of this process. Let's walk through a basic example by serializing `facebook/opt-125m` using the script, and then loading it for inference. @@ -25,7 +25,7 @@ CLI arguments. The docstring for the script itself explains the CLI args and how to use it properly in great detail, and we'll use one of the examples from the docstring directly, assuming we want to serialize and save our model at our S3 bucket example `s3://my-bucket`: ```bash -python examples/others/tensorize_vllm_model.py \ +python examples/features/tensorize_vllm_model.py \ --model facebook/opt-125m \ serialize \ --serialized-directory s3://my-bucket \ @@ -35,7 +35,7 @@ python examples/others/tensorize_vllm_model.py \ This saves the model tensors at `s3://my-bucket/vllm/facebook/opt-125m/v1`. If you intend on applying a LoRA adapter to your tensorized model, you can pass the HF id of the LoRA adapter in the above command, and the artifacts will be saved there too: ```bash -python examples/others/tensorize_vllm_model.py \ +python examples/features/tensorize_vllm_model.py \ --model facebook/opt-125m \ --lora-path \ serialize \ @@ -71,7 +71,7 @@ llm = LLM( As an example, CPU concurrency can be limited when serializing with `tensorizer` via the `limit_cpu_concurrency` parameter in the initializer for `TensorSerializer`. To set `limit_cpu_concurrency` to some arbitrary value, you would do so like this when serializing: ```bash -python examples/others/tensorize_vllm_model.py \ +python examples/features/tensorize_vllm_model.py \ --model facebook/opt-125m \ --lora-path \ serialize \ diff --git a/docs/serving/openai_compatible_server.md b/docs/serving/openai_compatible_server.md index bf1c153ec0..973c2ecdcb 100644 --- a/docs/serving/openai_compatible_server.md +++ b/docs/serving/openai_compatible_server.md @@ -251,7 +251,7 @@ The following extra parameters are supported: Our Responses API is compatible with [OpenAI's Responses API](https://platform.openai.com/docs/api-reference/responses); you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it. -Code example: [examples/online_serving/openai_responses_client_with_tools.py](../../examples/tool_calling/openai_responses_client_with_tools.py) +Code example: [examples/tool_calling/openai_responses_client_with_tools.py](../../examples/tool_calling/openai_responses_client_with_tools.py) #### Extra parameters @@ -456,8 +456,8 @@ Audio must be sent as base64-encoded PCM16 audio at 16kHz sample rate, mono chan #### Example Clients -- [openai_realtime_client.py](https://github.com/vllm-project/vllm/tree/main/examples/online_serving/openai_realtime_client.py) - Upload and transcribe an audio file -- [openai_realtime_microphone_client.py](https://github.com/vllm-project/vllm/tree/main/examples/online_serving/openai_realtime_microphone_client.py) - Gradio demo for live microphone transcription +- [openai_realtime_client.py](https://github.com/vllm-project/vllm/tree/main/examples/speech_to_text/realtime/openai_realtime_client.py) - Upload and transcribe an audio file +- [openai_realtime_microphone_client.py](https://github.com/vllm-project/vllm/tree/main/examples/speech_to_text/realtime/openai_realtime_microphone_client.py) - Gradio demo for live microphone transcription ### Tokenizer API diff --git a/examples/online_serving/api_client.py b/examples/applications/chatbot/api_client.py similarity index 100% rename from examples/online_serving/api_client.py rename to examples/applications/chatbot/api_client.py diff --git a/examples/online_serving/gradio_openai_chatbot_webserver.py b/examples/applications/chatbot/gradio_openai_chatbot_webserver.py similarity index 97% rename from examples/online_serving/gradio_openai_chatbot_webserver.py rename to examples/applications/chatbot/gradio_openai_chatbot_webserver.py index c76c60cc44..2a67aefc02 100644 --- a/examples/online_serving/gradio_openai_chatbot_webserver.py +++ b/examples/applications/chatbot/gradio_openai_chatbot_webserver.py @@ -5,7 +5,7 @@ Start vLLM API server: vllm serve meta-llama/Llama-2-7b-chat-hf Start Gradio OpenAI Chatbot Webserver: - python examples/online_serving/gradio_openai_chatbot_webserver.py \ + python examples/applications/chatbot/gradio_openai_chatbot_webserver.py \ -m meta-llama/Llama-2-7b-chat-hf Note that `pip install --upgrade gradio` is needed to run this example. diff --git a/examples/online_serving/gradio_webserver.py b/examples/applications/chatbot/gradio_webserver.py similarity index 97% rename from examples/online_serving/gradio_webserver.py rename to examples/applications/chatbot/gradio_webserver.py index 86d9ceb48b..f75636409c 100644 --- a/examples/online_serving/gradio_webserver.py +++ b/examples/applications/chatbot/gradio_webserver.py @@ -6,7 +6,7 @@ Start vLLM API server: --model meta-llama/Llama-2-7b-chat-hf Start Webserver: - python examples/online_serving/gradio_webserver.py + python examples/applications/chatbot/gradio_webserver.py Note that `pip install --upgrade gradio` is needed to run this example. More details: https://github.com/gradio-app/gradio diff --git a/examples/online_serving/streamlit_openai_chatbot_webserver.py b/examples/applications/chatbot/streamlit_openai_chatbot_webserver.py similarity index 100% rename from examples/online_serving/streamlit_openai_chatbot_webserver.py rename to examples/applications/chatbot/streamlit_openai_chatbot_webserver.py diff --git a/examples/online_serving/retrieval_augmented_generation_with_langchain.py b/examples/applications/rag/retrieval_augmented_generation_with_langchain.py similarity index 100% rename from examples/online_serving/retrieval_augmented_generation_with_langchain.py rename to examples/applications/rag/retrieval_augmented_generation_with_langchain.py diff --git a/examples/online_serving/retrieval_augmented_generation_with_llamaindex.py b/examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py similarity index 100% rename from examples/online_serving/retrieval_augmented_generation_with_llamaindex.py rename to examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py diff --git a/examples/offline_inference/async_llm_streaming.py b/examples/deployment/async_llm_streaming.py similarity index 98% rename from examples/offline_inference/async_llm_streaming.py rename to examples/deployment/async_llm_streaming.py index b876d536e3..ef69089a04 100644 --- a/examples/offline_inference/async_llm_streaming.py +++ b/examples/deployment/async_llm_streaming.py @@ -8,7 +8,7 @@ token-by-token output in offline inference scenarios. It demonstrates DELTA mode streaming where you receive new tokens as they are generated. Usage: - python examples/offline_inference/async_llm_streaming.py + python examples/deployment/async_llm_streaming.py """ import asyncio diff --git a/examples/online_serving/chart-helm/.helmignore b/examples/deployment/chart-helm/.helmignore similarity index 100% rename from examples/online_serving/chart-helm/.helmignore rename to examples/deployment/chart-helm/.helmignore diff --git a/examples/online_serving/chart-helm/Chart.yaml b/examples/deployment/chart-helm/Chart.yaml similarity index 100% rename from examples/online_serving/chart-helm/Chart.yaml rename to examples/deployment/chart-helm/Chart.yaml diff --git a/examples/online_serving/chart-helm/README.md b/examples/deployment/chart-helm/README.md similarity index 100% rename from examples/online_serving/chart-helm/README.md rename to examples/deployment/chart-helm/README.md diff --git a/examples/online_serving/chart-helm/ct.yaml b/examples/deployment/chart-helm/ct.yaml similarity index 100% rename from examples/online_serving/chart-helm/ct.yaml rename to examples/deployment/chart-helm/ct.yaml diff --git a/examples/online_serving/chart-helm/lintconf.yaml b/examples/deployment/chart-helm/lintconf.yaml similarity index 100% rename from examples/online_serving/chart-helm/lintconf.yaml rename to examples/deployment/chart-helm/lintconf.yaml diff --git a/examples/online_serving/chart-helm/templates/_helpers.tpl b/examples/deployment/chart-helm/templates/_helpers.tpl similarity index 100% rename from examples/online_serving/chart-helm/templates/_helpers.tpl rename to examples/deployment/chart-helm/templates/_helpers.tpl diff --git a/examples/online_serving/chart-helm/templates/configmap.yaml b/examples/deployment/chart-helm/templates/configmap.yaml similarity index 100% rename from examples/online_serving/chart-helm/templates/configmap.yaml rename to examples/deployment/chart-helm/templates/configmap.yaml diff --git a/examples/online_serving/chart-helm/templates/custom-objects.yaml b/examples/deployment/chart-helm/templates/custom-objects.yaml similarity index 100% rename from examples/online_serving/chart-helm/templates/custom-objects.yaml rename to examples/deployment/chart-helm/templates/custom-objects.yaml diff --git a/examples/online_serving/chart-helm/templates/deployment.yaml b/examples/deployment/chart-helm/templates/deployment.yaml similarity index 100% rename from examples/online_serving/chart-helm/templates/deployment.yaml rename to examples/deployment/chart-helm/templates/deployment.yaml diff --git a/examples/online_serving/chart-helm/templates/hpa.yaml b/examples/deployment/chart-helm/templates/hpa.yaml similarity index 100% rename from examples/online_serving/chart-helm/templates/hpa.yaml rename to examples/deployment/chart-helm/templates/hpa.yaml diff --git a/examples/online_serving/chart-helm/templates/job.yaml b/examples/deployment/chart-helm/templates/job.yaml similarity index 100% rename from examples/online_serving/chart-helm/templates/job.yaml rename to examples/deployment/chart-helm/templates/job.yaml diff --git a/examples/online_serving/chart-helm/templates/poddisruptionbudget.yaml b/examples/deployment/chart-helm/templates/poddisruptionbudget.yaml similarity index 100% rename from examples/online_serving/chart-helm/templates/poddisruptionbudget.yaml rename to examples/deployment/chart-helm/templates/poddisruptionbudget.yaml diff --git a/examples/online_serving/chart-helm/templates/pvc.yaml b/examples/deployment/chart-helm/templates/pvc.yaml similarity index 100% rename from examples/online_serving/chart-helm/templates/pvc.yaml rename to examples/deployment/chart-helm/templates/pvc.yaml diff --git a/examples/online_serving/chart-helm/templates/secrets.yaml b/examples/deployment/chart-helm/templates/secrets.yaml similarity index 100% rename from examples/online_serving/chart-helm/templates/secrets.yaml rename to examples/deployment/chart-helm/templates/secrets.yaml diff --git a/examples/online_serving/chart-helm/templates/service.yaml b/examples/deployment/chart-helm/templates/service.yaml similarity index 100% rename from examples/online_serving/chart-helm/templates/service.yaml rename to examples/deployment/chart-helm/templates/service.yaml diff --git a/examples/online_serving/chart-helm/tests/deployment_test.yaml b/examples/deployment/chart-helm/tests/deployment_test.yaml similarity index 100% rename from examples/online_serving/chart-helm/tests/deployment_test.yaml rename to examples/deployment/chart-helm/tests/deployment_test.yaml diff --git a/examples/online_serving/chart-helm/tests/job_test.yaml b/examples/deployment/chart-helm/tests/job_test.yaml similarity index 100% rename from examples/online_serving/chart-helm/tests/job_test.yaml rename to examples/deployment/chart-helm/tests/job_test.yaml diff --git a/examples/online_serving/chart-helm/tests/pvc_test.yaml b/examples/deployment/chart-helm/tests/pvc_test.yaml similarity index 100% rename from examples/online_serving/chart-helm/tests/pvc_test.yaml rename to examples/deployment/chart-helm/tests/pvc_test.yaml diff --git a/examples/online_serving/chart-helm/values.schema.json b/examples/deployment/chart-helm/values.schema.json similarity index 100% rename from examples/online_serving/chart-helm/values.schema.json rename to examples/deployment/chart-helm/values.schema.json diff --git a/examples/online_serving/chart-helm/values.yaml b/examples/deployment/chart-helm/values.yaml similarity index 100% rename from examples/online_serving/chart-helm/values.yaml rename to examples/deployment/chart-helm/values.yaml diff --git a/examples/offline_inference/llm_engine_example.py b/examples/deployment/llm_engine_example.py similarity index 100% rename from examples/offline_inference/llm_engine_example.py rename to examples/deployment/llm_engine_example.py diff --git a/examples/online_serving/sagemaker-entrypoint.sh b/examples/deployment/sagemaker-entrypoint.sh similarity index 100% rename from examples/online_serving/sagemaker-entrypoint.sh rename to examples/deployment/sagemaker-entrypoint.sh diff --git a/examples/online_serving/disaggregated_serving/disagg_proxy_multiturn.py b/examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py similarity index 100% rename from examples/online_serving/disaggregated_serving/disagg_proxy_multiturn.py rename to examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py diff --git a/examples/others/logging_configuration.md b/examples/features/logging_configuration.md similarity index 100% rename from examples/others/logging_configuration.md rename to examples/features/logging_configuration.md diff --git a/examples/others/tensorize_vllm_model.py b/examples/features/tensorize_vllm_model.py similarity index 97% rename from examples/others/tensorize_vllm_model.py rename to examples/features/tensorize_vllm_model.py index 3644a03b32..a89b178126 100644 --- a/examples/others/tensorize_vllm_model.py +++ b/examples/features/tensorize_vllm_model.py @@ -33,7 +33,7 @@ https://github.com/coreweave/tensorizer To serialize a model, install vLLM from source, then run something like this from the root level of this repository: -python examples/others/tensorize_vllm_model.py \ +python examples/features/tensorize_vllm_model.py \ --model facebook/opt-125m \ serialize \ --serialized-directory s3://my-bucket \ @@ -53,7 +53,7 @@ providing a `--keyfile` argument. To deserialize a model, you can run something like this from the root level of this repository: -python examples/others/tensorize_vllm_model.py \ +python examples/features/tensorize_vllm_model.py \ --model EleutherAI/gpt-j-6B \ --dtype float16 \ deserialize \ @@ -71,11 +71,11 @@ shard's rank. Sharded models serialized with this script will be named as model-rank-%03d.tensors For more information on the available arguments for serializing, run -`python -m examples.others.tensorize_vllm_model serialize --help`. +`python -m examples.features.tensorize_vllm_model serialize --help`. Or for deserializing: -`python examples/others/tensorize_vllm_model.py deserialize --help`. +`python examples/features/tensorize_vllm_model.py deserialize --help`. Once a model is serialized, tensorizer can be invoked with the `LLM` class directly to load models: @@ -100,7 +100,7 @@ vllm serve s3://my-bucket/vllm/facebook/opt-125m/v1 \ In order to see all of the available arguments usable to configure loading with tensorizer that are given to `TensorizerConfig`, run: -`python examples/others/tensorize_vllm_model.py deserialize --help` +`python examples/features/tensorize_vllm_model.py deserialize --help` under the `tensorizer options` section. These can also be used for deserialization in this example script, although `--tensorizer-uri` and diff --git a/examples/observability/dashboards/README.md b/examples/observability/dashboards/README.md index e5f5010a42..29ec932cef 100644 --- a/examples/observability/dashboards/README.md +++ b/examples/observability/dashboards/README.md @@ -43,7 +43,7 @@ Both platforms provide equivalent monitoring capabilities: First, navigate to this example's directory: ```bash -cd examples/online_serving/dashboards +cd examples/observability/dashboards ``` ### Grafana diff --git a/examples/online_serving/utils.py b/examples/online_serving/utils.py deleted file mode 100644 index a512d8a31b..0000000000 --- a/examples/online_serving/utils.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from openai import APIConnectionError, OpenAI -from openai.pagination import SyncPage -from openai.types.model import Model - - -def get_first_model(client: OpenAI) -> str: - """ - Get the first model from the vLLM server. - """ - try: - models: SyncPage[Model] = client.models.list() - except APIConnectionError as e: - raise RuntimeError( - "Failed to get the list of models from the vLLM server at " - f"{client.base_url} with API key {client.api_key}. Check\n" - "1. the server is running\n" - "2. the server URL is correct\n" - "3. the API key is correct" - ) from e - - if len(models.data) == 0: - raise RuntimeError(f"No models found on the vLLM server at {client.base_url}") - - return models.data[0].id diff --git a/examples/pooling/embed/openai_embedding_long_text/service.sh b/examples/pooling/embed/openai_embedding_long_text/service.sh index 37a8b625b7..68950d04ee 100644 --- a/examples/pooling/embed/openai_embedding_long_text/service.sh +++ b/examples/pooling/embed/openai_embedding_long_text/service.sh @@ -119,7 +119,7 @@ echo " - API Key: $API_KEY" echo " - Native Pooling: $POOLING_TYPE | Cross-chunk: MEAN" echo "" echo "๐Ÿงช Test the server with:" -echo " python examples/online_serving/openai_embedding_long_text/client.py" +echo " python examples/pooling/embed/openai_embedding_long_text/client.py" echo "" echo "๐Ÿ“š Enhanced features enabled:" echo " โœ… Intelligent native pooling type detection" diff --git a/examples/speech_to_text/lid/openai_lid_client.py b/examples/speech_to_text/lid/openai_lid_client.py index 0ce0fbc922..d91df3298c 100644 --- a/examples/speech_to_text/lid/openai_lid_client.py +++ b/examples/speech_to_text/lid/openai_lid_client.py @@ -15,14 +15,14 @@ Start the vLLM server: Then run this script: # Use the built-in sample audio - python examples/online_serving/openai_lid_client.py + python examples/speech_to_text/lid/openai_lid_client.py # Use your own audio file(s) - python examples/online_serving/openai_lid_client.py \ + python examples/speech_to_text/lid/openai_lid_client.py \ --audio_paths audio_en.wav audio_zh.wav audio_fr.wav # Batch-identify multiple files in one run - python examples/online_serving/openai_lid_client.py \ + python examples/speech_to_text/lid/openai_lid_client.py \ --audio_paths /path/to/dir/*.wav Requirements: diff --git a/requirements/common.txt b/requirements/common.txt index 68f5e165b9..8431703252 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -41,7 +41,7 @@ compressed-tensors == 0.15.0.1 # required for compressed-tensors depyf==0.20.0 # required for profiling and debugging with compilation config cloudpickle # allows pickling lambda functions in model_executor/models/registry.py watchfiles # required for http server to monitor the updates of TLS files -python-json-logger # Used by logging as per examples/others/logging_configuration.md +python-json-logger # Used by logging as per examples/features/logging_configuration.md ninja # Required for xgrammar, rocm, tpu, xpu pybase64 # fast base64 implementation cbor2 # Required for cross-language serialization of hashable objects diff --git a/tests/lora/test_llama_tp.py b/tests/lora/test_llama_tp.py index abcce66fa9..99c823238d 100644 --- a/tests/lora/test_llama_tp.py +++ b/tests/lora/test_llama_tp.py @@ -184,7 +184,7 @@ def test_tp2_serialize_and_deserialize_lora( result = subprocess.run( [ sys.executable, - f"{VLLM_PATH}/examples/others/tensorize_vllm_model.py", + f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py", "--model", MODEL_PATH, "--lora-path", diff --git a/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py b/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py index 3b950c843c..a15a624c90 100644 --- a/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py +++ b/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py @@ -460,7 +460,7 @@ async def test_serialize_and_serve_entrypoints(tmp_path): result = subprocess.run( [ sys.executable, - f"{VLLM_PATH}/examples/others/tensorize_vllm_model.py", + f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py", "--model", model_ref, "serialize", diff --git a/vllm/assets/video.py b/vllm/assets/video.py index f5e443db97..9ec2e4d167 100644 --- a/vllm/assets/video.py +++ b/vllm/assets/video.py @@ -139,6 +139,6 @@ class VideoAsset: """ Read audio data from the video asset, used in Qwen2.5-Omni examples. - See also: examples/offline_inference/qwen2_5_omni/only_thinker.py + See also: examples/generate/multimodal/qwen2_5_omni/only_thinker.py """ return load_audio_pyav(self.video_path, sr=sampling_rate)[0] diff --git a/vllm/model_executor/model_loader/tensorizer.py b/vllm/model_executor/model_loader/tensorizer.py index 3e6ed248ff..37d37d55f5 100644 --- a/vllm/model_executor/model_loader/tensorizer.py +++ b/vllm/model_executor/model_loader/tensorizer.py @@ -211,7 +211,7 @@ class TensorizerConfig(MutableMapping): encryption_keyfile: File path to a binary file containing a binary key to use for decryption. `None` (the default) means no decryption. See the example script in - examples/others/tensorize_vllm_model.py. + examples/features/tensorize_vllm_model.py. s3_access_key_id: The access key for the S3 bucket. Can also be set via the S3_ACCESS_KEY_ID environment variable. s3_secret_access_key: The secret access key for the S3 bucket. Can also @@ -579,7 +579,7 @@ def tensorizer_weights_iterator( "loading on vLLM, as tensorizer is forced to load to CPU. " "Consider deserializing a vLLM model instead for faster " "load times. See the " - "examples/others/tensorize_vllm_model.py example script " + "examples/features/tensorize_vllm_model.py example script " "for serializing vLLM models." ) diff --git a/vllm/model_executor/model_loader/tensorizer_loader.py b/vllm/model_executor/model_loader/tensorizer_loader.py index c5bff13129..338f9eac07 100644 --- a/vllm/model_executor/model_loader/tensorizer_loader.py +++ b/vllm/model_executor/model_loader/tensorizer_loader.py @@ -73,7 +73,7 @@ class TensorizerLoader(BaseModelLoader): """Load a serialized model with tensorizer to the CPU. This is only necessary when the model isn't vLLM-tensorized (see - examples/others/tensorize_vllm_model.py) This should still + examples/features/tensorize_vllm_model.py) This should still be faster than default HuggingFace loading, but will be slower than loading a vLLM-tensorized model. """ @@ -104,7 +104,7 @@ class TensorizerLoader(BaseModelLoader): """Load serialized model weights with tensorizer. Expects a vLLM-tensorized model. See the - examples/others/tensorize_vllm_model.py example script + examples/features/tensorize_vllm_model.py example script for serializing vLLM models.""" if is_vllm_tensorized(self.tensorizer_config): tensorizer_config = self._patch_tensorizer_config(model_config)