From 51c1ee9b7c8acbba4899a8ebffd390685d171946 Mon Sep 17 00:00:00 2001 From: "wang.yuqi" Date: Wed, 6 May 2026 16:20:38 +0800 Subject: [PATCH] [Examples] Resettle Disaggregated examples. (#40759) Signed-off-by: wang.yuqi Signed-off-by: wang.yuqi Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .github/mergify.yml | 4 +--- docs/deployment/frameworks/anyscale.md | 2 +- docs/design/p2p_nccl_connector.md | 4 ++-- docs/features/disagg_encoder.md | 4 ++-- docs/features/disagg_prefill.md | 12 ++++++------ docs/features/mooncake_connector_usage.md | 6 +++--- docs/serving/distributed_troubleshooting.md | 4 ++-- docs/serving/openai_compatible_server.md | 2 +- docs/serving/parallelism_scaling.md | 4 ++-- .../disaggregated_encoder/README.md | 0 .../disaggregated_encoder/disagg_1e1p1d_example.sh | 0 .../disaggregated_encoder/disagg_1e1pd_example.sh | 0 .../disaggregated_encoder/disagg_epd_proxy.py | 0 .../disaggregated_prefill.py | 0 .../disaggregated_prefill.sh | 0 .../disaggregated_serving/README.md | 0 .../disaggregated_serving/disagg_proxy_demo.py | 2 +- .../disaggregated_serving/example_mm_serve.py | 0 .../disaggregated_serving/kv_events.sh | 0 .../disaggregated_serving/moriio_toy_proxy_server.py | 0 .../ec_both_encoder/ec_both_encoder.sh | 0 .../example_connector}/README.md | 2 +- .../example_connector}/decode_example.py | 0 .../example_connector}/prefill_example.py | 0 .../example_connector}/run.sh | 0 .../flexkv_connector}/prefix_caching_flexkv.py | 2 +- .../kv_load_failure_recovery_offline}/README.md | 4 ++-- .../decode_example.py | 0 .../load_recovery_example_connector.py | 0 .../prefill_example.py | 0 .../kv_load_failure_recovery_offline}/run.sh | 0 examples/{others => disaggregated}/lmcache/README.md | 0 .../lmcache/cpu_offload_lmcache.py | 0 .../lmcache/disagg_prefill_lmcache_v0.py | 0 .../configs/lmcache-decoder-config.yaml | 0 .../configs/lmcache-prefiller-config.yaml | 0 .../disagg_prefill_lmcache_v1/disagg_example_nixl.sh | 0 .../disagg_prefill_lmcache_v1/disagg_proxy_server.py | 0 .../disagg_vllm_launcher.sh | 0 .../lmcache/kv_cache_sharing_lmcache_v1.py | 0 .../mooncake_connector/mooncake_connector_proxy.py | 0 .../mooncake_connector/run_mooncake_connector.sh | 0 .../p2p_nccl_xpyd}/disagg_example_p2p_nccl_xpyd.sh | 0 .../p2p_nccl_xpyd}/disagg_proxy_p2p_nccl_xpyd.py | 0 .../batch_llm_inference.py | 0 .../elastic_ep/bench.sh | 0 .../elastic_ep/scale.py | 0 .../elastic_ep/serve_deepseek_v2.sh | 0 .../multi-node-serving.sh | 0 .../ray_serve_deepseek.py | 0 .../{online_serving => ray_serving}/run_cluster.sh | 0 tests/v1/ec_connector/integration/README.md | 2 +- .../integration/run_epd_correctness_test.sh | 4 ++-- vllm/distributed/kv_transfer/README.md | 2 +- 54 files changed, 29 insertions(+), 31 deletions(-) rename examples/{online_serving => disaggregated}/disaggregated_encoder/README.md (100%) rename examples/{online_serving => disaggregated}/disaggregated_encoder/disagg_1e1p1d_example.sh (100%) rename examples/{online_serving => disaggregated}/disaggregated_encoder/disagg_1e1pd_example.sh (100%) rename examples/{online_serving => disaggregated}/disaggregated_encoder/disagg_epd_proxy.py (100%) rename examples/{offline_inference => disaggregated}/disaggregated_prefill.py (100%) rename examples/{online_serving => disaggregated}/disaggregated_prefill.sh (100%) rename examples/{online_serving => disaggregated}/disaggregated_serving/README.md (100%) rename examples/{online_serving => disaggregated}/disaggregated_serving/disagg_proxy_demo.py (99%) rename examples/{online_serving => disaggregated}/disaggregated_serving/example_mm_serve.py (100%) rename examples/{online_serving => disaggregated}/disaggregated_serving/kv_events.sh (100%) rename examples/{online_serving => disaggregated}/disaggregated_serving/moriio_toy_proxy_server.py (100%) rename examples/{online_serving => disaggregated}/ec_both_encoder/ec_both_encoder.sh (100%) mode change 100755 => 100644 rename examples/{offline_inference/disaggregated-prefill-v1 => disaggregated/example_connector}/README.md (81%) rename examples/{offline_inference/disaggregated-prefill-v1 => disaggregated/example_connector}/decode_example.py (100%) rename examples/{offline_inference/disaggregated-prefill-v1 => disaggregated/example_connector}/prefill_example.py (100%) rename examples/{offline_inference/disaggregated-prefill-v1 => disaggregated/example_connector}/run.sh (100%) rename examples/{offline_inference => disaggregated/flexkv_connector}/prefix_caching_flexkv.py (98%) rename examples/{offline_inference/kv_load_failure_recovery => disaggregated/kv_load_failure_recovery_offline}/README.md (91%) rename examples/{offline_inference/kv_load_failure_recovery => disaggregated/kv_load_failure_recovery_offline}/decode_example.py (100%) rename examples/{offline_inference/kv_load_failure_recovery => disaggregated/kv_load_failure_recovery_offline}/load_recovery_example_connector.py (100%) rename examples/{offline_inference/kv_load_failure_recovery => disaggregated/kv_load_failure_recovery_offline}/prefill_example.py (100%) rename examples/{offline_inference/kv_load_failure_recovery => disaggregated/kv_load_failure_recovery_offline}/run.sh (100%) mode change 100755 => 100644 rename examples/{others => disaggregated}/lmcache/README.md (100%) rename examples/{others => disaggregated}/lmcache/cpu_offload_lmcache.py (100%) rename examples/{others => disaggregated}/lmcache/disagg_prefill_lmcache_v0.py (100%) rename examples/{others => disaggregated}/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-decoder-config.yaml (100%) rename examples/{others => disaggregated}/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-prefiller-config.yaml (100%) rename examples/{others => disaggregated}/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh (100%) rename examples/{others => disaggregated}/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py (100%) rename examples/{others => disaggregated}/lmcache/disagg_prefill_lmcache_v1/disagg_vllm_launcher.sh (100%) rename examples/{others => disaggregated}/lmcache/kv_cache_sharing_lmcache_v1.py (100%) rename examples/{online_serving/disaggregated_serving => disaggregated}/mooncake_connector/mooncake_connector_proxy.py (100%) rename examples/{online_serving/disaggregated_serving => disaggregated}/mooncake_connector/run_mooncake_connector.sh (100%) rename examples/{online_serving/disaggregated_serving_p2p_nccl_xpyd => disaggregated/p2p_nccl_xpyd}/disagg_example_p2p_nccl_xpyd.sh (100%) rename examples/{online_serving/disaggregated_serving_p2p_nccl_xpyd => disaggregated/p2p_nccl_xpyd}/disagg_proxy_p2p_nccl_xpyd.py (100%) rename examples/{offline_inference => ray_serving}/batch_llm_inference.py (100%) rename examples/{online_serving => ray_serving}/elastic_ep/bench.sh (100%) rename examples/{online_serving => ray_serving}/elastic_ep/scale.py (100%) rename examples/{online_serving => ray_serving}/elastic_ep/serve_deepseek_v2.sh (100%) rename examples/{online_serving => ray_serving}/multi-node-serving.sh (100%) rename examples/{online_serving => ray_serving}/ray_serve_deepseek.py (100%) rename examples/{online_serving => ray_serving}/run_cluster.sh (100%) diff --git a/.github/mergify.yml b/.github/mergify.yml index de3c76fd45..2d36e35070 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -477,9 +477,7 @@ pull_request_rules: conditions: - label != stale - or: - - files~=^examples/online_serving/disaggregated[^/]*/.* - - files~=^examples/offline_inference/disaggregated[^/]*/.* - - files~=^examples/others/lmcache/ + - files~=^examples/disaggregated/ - files~=^tests/v1/kv_connector/ - files~=^vllm/distributed/kv_transfer/ - title~=(?i)\bP/?D\b diff --git a/docs/deployment/frameworks/anyscale.md b/docs/deployment/frameworks/anyscale.md index 965742ec07..6888e4dbf0 100644 --- a/docs/deployment/frameworks/anyscale.md +++ b/docs/deployment/frameworks/anyscale.md @@ -3,7 +3,7 @@ [Anyscale](https://www.anyscale.com) is a managed, multi-cloud platform developed by the creators of Ray. Anyscale automates the entire lifecycle of Ray clusters in your AWS, GCP, or Azure account, delivering the flexibility of open-source Ray -without the operational overhead of maintaining Kubernetes control planes, configuring autoscalers, managing observability stacks, or manually managing head and worker nodes with helper scripts like [examples/online_serving/run_cluster.sh](../../../examples/online_serving/run_cluster.sh). +without the operational overhead of maintaining Kubernetes control planes, configuring autoscalers, managing observability stacks, or manually managing head and worker nodes with helper scripts like [examples/ray_serving/run_cluster.sh](../../../examples/ray_serving/run_cluster.sh). When serving large language models with vLLM, Anyscale can rapidly provision [production-ready HTTPS endpoints](https://docs.anyscale.com/examples/deploy-ray-serve-llms) or [fault-tolerant batch inference jobs](https://docs.anyscale.com/examples/ray-data-llm). diff --git a/docs/design/p2p_nccl_connector.md b/docs/design/p2p_nccl_connector.md index 4674bef8d2..c1de955b6f 100644 --- a/docs/design/p2p_nccl_connector.md +++ b/docs/design/p2p_nccl_connector.md @@ -88,7 +88,7 @@ pip install "vllm>=0.9.2" #### Proxy (e.g. 10.0.1.1) ```shell -cd {your vllm directory}/examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/ +cd {your vllm directory}/examples/disaggregated/p2p_nccl_xpyd/ python3 disagg_proxy_p2p_nccl_xpyd.py & ``` @@ -181,7 +181,7 @@ python3 disagg_proxy_p2p_nccl_xpyd.py & #### Proxy (e.g. 10.0.1.1) ```shell -cd {your vllm directory}/examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/ +cd {your vllm directory}/examples/disaggregated/p2p_nccl_xpyd/ python3 disagg_proxy_p2p_nccl_xpyd.py & ``` diff --git a/docs/features/disagg_encoder.md b/docs/features/disagg_encoder.md index af6da94aa6..c27d6b2772 100644 --- a/docs/features/disagg_encoder.md +++ b/docs/features/disagg_encoder.md @@ -36,10 +36,10 @@ The current reference pathway is **ExampleConnector**. Below ready-to-run scripts shows the workflow: 1 Encoder instance + 1 PD instance: -`examples/online_serving/disaggregated_encoder/disagg_1e1pd_example.sh` +`examples/disaggregated/disaggregated_encoder/disagg_1e1pd_example.sh` 1 Encoder instance + 1 Prefill instance + 1 Decode instance: -`examples/online_serving/disaggregated_encoder/disagg_1e1p1d_example.sh` +`examples/disaggregated/disaggregated_encoder/disagg_1e1p1d_example.sh` --- diff --git a/docs/features/disagg_prefill.md b/docs/features/disagg_prefill.md index 5167d61239..9ad005be3f 100644 --- a/docs/features/disagg_prefill.md +++ b/docs/features/disagg_prefill.md @@ -17,15 +17,15 @@ Two main reasons: ## Usage example -Please refer to [examples/online_serving/disaggregated_prefill.sh](../../examples/online_serving/disaggregated_prefill.sh) for the example usage of disaggregated prefilling. +Please refer to [examples/disaggregated/disaggregated_prefill.sh](../../examples/disaggregated/disaggregated_prefill.sh) for the example usage of disaggregated prefilling. Now supports 6 types of connectors: -- **ExampleConnector**: refer to [examples/offline_inference/disaggregated-prefill-v1/run.sh](../../examples/offline_inference/disaggregated-prefill-v1/run.sh) for the example usage of ExampleConnector disaggregated prefilling. -- **LMCacheConnectorV1**: refer to [examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh](../../examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh) for the example usage of LMCacheConnectorV1 disaggregated prefilling which uses NIXL as the underlying KV transmission. +- **ExampleConnector**: refer to [examples/disaggregated/example_connector/run.sh](../../examples/disaggregated/example_connector/run.sh) for the example usage of ExampleConnector disaggregated prefilling. +- **LMCacheConnectorV1**: refer to [examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh](../../examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh) for the example usage of LMCacheConnectorV1 disaggregated prefilling which uses NIXL as the underlying KV transmission. - **NixlConnector**: refer to [tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh](../../tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh) for the example usage of NixlConnector disaggregated prefilling which support fully async send/recv. For detailed usage guide, see [NixlConnector Usage Guide](nixl_connector_usage.md). For feature compatibility details, see [NixlConnector Compatibility Matrix](nixl_connector_compatibility.md). -- **P2pNcclConnector**: refer to [examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh](../../examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh) for the example usage of P2pNcclConnector disaggregated prefilling. -- **MooncakeConnector**: refer to [examples/online_serving/disaggregated_serving/mooncake_connector/run_mooncake_connector.sh](../../examples/online_serving/disaggregated_serving/mooncake_connector/run_mooncake_connector.sh) for the example usage of ExampleConnector disaggregated prefilling. For detailed usage guide, see [MooncakeConnector Usage Guide](mooncake_connector_usage.md). +- **P2pNcclConnector**: refer to [examples/disaggregated/p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh](../../examples/disaggregated/p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh) for the example usage of P2pNcclConnector disaggregated prefilling. +- **MooncakeConnector**: refer to [examples/disaggregated/mooncake_connector/run_mooncake_connector.sh](../../examples/disaggregated/mooncake_connector/run_mooncake_connector.sh) for the example usage of MooncakeConnector disaggregated prefilling. For detailed usage guide, see [MooncakeConnector Usage Guide](mooncake_connector_usage.md). - **MultiConnector**: take advantage of the kv_connector_extra_config: dict[str, Any] already present in KVTransferConfig to stash all the connectors we want in an ordered list of kwargs.such as: ```bash @@ -44,7 +44,7 @@ For NixlConnector, you may also specify one or multiple NIXL_Backend. Such as: --kv-transfer-config '{"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"block_size": 64, "cpu_bytes_to_use": 1000000000}}' ``` -- **FlexKVConnectorV1**: refer to [examples/offline_inference/prefix_caching_flexkv.py](../../examples/offline_inference/prefix_caching_flexkv.py) for the example usage of FlexKVConnectorV1. FlexKV is a distributed KV Store and multi-level cache management system for ultra-large-scale LLM inference. +- **FlexKVConnectorV1**: refer to [examples/disaggregated/flexkv_connector/prefix_caching_flexkv.py](../../examples/disaggregated/flexkv_connector/prefix_caching_flexkv.py) for the example usage of FlexKVConnectorV1. FlexKV is a distributed KV Store and multi-level cache management system for ultra-large-scale LLM inference. ```bash --kv-transfer-config '{"kv_connector":"FlexKVConnectorV1","kv_role":"kv_both"}' diff --git a/docs/features/mooncake_connector_usage.md b/docs/features/mooncake_connector_usage.md index 0e2478924e..cc8c8ecff2 100644 --- a/docs/features/mooncake_connector_usage.md +++ b/docs/features/mooncake_connector_usage.md @@ -31,7 +31,7 @@ vllm serve Qwen/Qwen2.5-7B-Instruct --port 8020 --kv-transfer-config '{"kv_conne ### Proxy ```bash -python examples/online_serving/disaggregated_serving/mooncake_connector/mooncake_connector_proxy.py --prefill http://192.168.0.2:8010 --decode http://192.168.0.3:8020 +python examples/disaggregated/disaggregated_serving/mooncake_connector/mooncake_connector_proxy.py --prefill http://192.168.0.2:8010 --decode http://192.168.0.3:8020 ``` Now you can send requests to the proxy server through port 8000. @@ -65,5 +65,5 @@ Now you can send requests to the proxy server through port 8000. Refer to these example scripts in the vLLM repository: -- [run_mooncake_connector.sh](../../examples/online_serving/disaggregated_serving/mooncake_connector/run_mooncake_connector.sh) -- [mooncake_connector_proxy.py](../../examples/online_serving/disaggregated_serving/mooncake_connector/mooncake_connector_proxy.py) +- [run_mooncake_connector.sh](../../examples/disaggregated/mooncake_connector/run_mooncake_connector.sh) +- [mooncake_connector_proxy.py](../../examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py) diff --git a/docs/serving/distributed_troubleshooting.md b/docs/serving/distributed_troubleshooting.md index b5354a7e55..e6dde49442 100644 --- a/docs/serving/distributed_troubleshooting.md +++ b/docs/serving/distributed_troubleshooting.md @@ -4,11 +4,11 @@ For general troubleshooting, see [Troubleshooting](../usage/troubleshooting.md). ## Verify inter-node GPU communication -After you start the Ray cluster, verify GPU-to-GPU communication across nodes. Proper configuration can be non-trivial. For more information, see [troubleshooting script](../usage/troubleshooting.md#incorrect-hardwaredriver). If you need additional environment variables for communication configuration, append them to [examples/online_serving/run_cluster.sh](../../examples/online_serving/run_cluster.sh), for example `-e NCCL_SOCKET_IFNAME=eth0`. Setting environment variables during cluster creation is recommended because the variables propagate to all nodes. In contrast, setting environment variables in the shell affects only the local node. For more information, see . +After you start the Ray cluster, verify GPU-to-GPU communication across nodes. Proper configuration can be non-trivial. For more information, see [troubleshooting script](../usage/troubleshooting.md#incorrect-hardwaredriver). If you need additional environment variables for communication configuration, append them to [examples/ray_serving/run_cluster.sh](../../examples/ray_serving/run_cluster.sh), for example `-e NCCL_SOCKET_IFNAME=eth0`. Setting environment variables during cluster creation is recommended because the variables propagate to all nodes. In contrast, setting environment variables in the shell affects only the local node. For more information, see . ## No available node types can fulfill resource request -The error message `Error: No available node types can fulfill resource request` can appear even when the cluster has enough GPUs. The issue often occurs when nodes have multiple IP addresses and vLLM can't select the correct one. Ensure that vLLM and Ray use the same IP address by setting `VLLM_HOST_IP` in [examples/online_serving/run_cluster.sh](../../examples/online_serving/run_cluster.sh) (with a different value on each node). Use `ray status` and `ray list nodes` to verify the chosen IP address. For more information, see . +The error message `Error: No available node types can fulfill resource request` can appear even when the cluster has enough GPUs. The issue often occurs when nodes have multiple IP addresses and vLLM can't select the correct one. Ensure that vLLM and Ray use the same IP address by setting `VLLM_HOST_IP` in [examples/ray_serving/run_cluster.sh](../../examples/ray_serving/run_cluster.sh) (with a different value on each node). Use `ray status` and `ray list nodes` to verify the chosen IP address. For more information, see . ## Ray observability diff --git a/docs/serving/openai_compatible_server.md b/docs/serving/openai_compatible_server.md index 59f02a0065..bf1c153ec0 100644 --- a/docs/serving/openai_compatible_server.md +++ b/docs/serving/openai_compatible_server.md @@ -542,6 +542,6 @@ Key capabilities: - Scales from a single GPU to a multi-node cluster without code changes. - Provides observability and autoscaling policies through Ray dashboards and metrics. -The following example shows how to deploy a large model like DeepSeek R1 with Ray Serve LLM: [examples/online_serving/ray_serve_deepseek.py](../../examples/online_serving/ray_serve_deepseek.py). +The following example shows how to deploy a large model like DeepSeek R1 with Ray Serve LLM: [examples/ray_serving/ray_serve_deepseek.py](../../examples/ray_serving/ray_serve_deepseek.py). Learn more about Ray Serve LLM with the official [Ray Serve LLM documentation](https://docs.ray.io/en/latest/serve/llm/index.html). diff --git a/docs/serving/parallelism_scaling.md b/docs/serving/parallelism_scaling.md index b69ca17e83..0f86a25672 100644 --- a/docs/serving/parallelism_scaling.md +++ b/docs/serving/parallelism_scaling.md @@ -78,7 +78,7 @@ For details, see the [Ray documentation](https://docs.ray.io/en/latest/index.htm ### Ray cluster setup with containers -The helper script [examples/online_serving/run_cluster.sh](../../examples/online_serving/run_cluster.sh) starts containers across nodes and initializes Ray. By default, the script runs Docker without administrative privileges, which prevents access to the GPU performance counters when profiling or tracing. To enable admin privileges, add the `--cap-add=CAP_SYS_ADMIN` flag to the Docker command. +The helper script [examples/ray_serving/run_cluster.sh](../../examples/ray_serving/run_cluster.sh) starts containers across nodes and initializes Ray. By default, the script runs Docker without administrative privileges, which prevents access to the GPU performance counters when profiling or tracing. To enable admin privileges, add the `--cap-add=CAP_SYS_ADMIN` flag to the Docker command. Choose one node as the head node and run: @@ -162,7 +162,7 @@ vllm serve /path/to/the/model/in/the/container \ Efficient tensor parallelism requires fast internode communication, preferably through high-speed network adapters such as InfiniBand. To set up the cluster to use InfiniBand, append additional arguments like `--privileged -e NCCL_IB_HCA=mlx5` to the -[examples/online_serving/run_cluster.sh](../../examples/online_serving/run_cluster.sh) helper script. +[examples/ray_serving/run_cluster.sh](../../examples/ray_serving/run_cluster.sh) helper script. Contact your system administrator for more information about the required flags. ## Enabling GPUDirect RDMA diff --git a/examples/online_serving/disaggregated_encoder/README.md b/examples/disaggregated/disaggregated_encoder/README.md similarity index 100% rename from examples/online_serving/disaggregated_encoder/README.md rename to examples/disaggregated/disaggregated_encoder/README.md diff --git a/examples/online_serving/disaggregated_encoder/disagg_1e1p1d_example.sh b/examples/disaggregated/disaggregated_encoder/disagg_1e1p1d_example.sh similarity index 100% rename from examples/online_serving/disaggregated_encoder/disagg_1e1p1d_example.sh rename to examples/disaggregated/disaggregated_encoder/disagg_1e1p1d_example.sh diff --git a/examples/online_serving/disaggregated_encoder/disagg_1e1pd_example.sh b/examples/disaggregated/disaggregated_encoder/disagg_1e1pd_example.sh similarity index 100% rename from examples/online_serving/disaggregated_encoder/disagg_1e1pd_example.sh rename to examples/disaggregated/disaggregated_encoder/disagg_1e1pd_example.sh diff --git a/examples/online_serving/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py similarity index 100% rename from examples/online_serving/disaggregated_encoder/disagg_epd_proxy.py rename to examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py diff --git a/examples/offline_inference/disaggregated_prefill.py b/examples/disaggregated/disaggregated_prefill.py similarity index 100% rename from examples/offline_inference/disaggregated_prefill.py rename to examples/disaggregated/disaggregated_prefill.py diff --git a/examples/online_serving/disaggregated_prefill.sh b/examples/disaggregated/disaggregated_prefill.sh similarity index 100% rename from examples/online_serving/disaggregated_prefill.sh rename to examples/disaggregated/disaggregated_prefill.sh diff --git a/examples/online_serving/disaggregated_serving/README.md b/examples/disaggregated/disaggregated_serving/README.md similarity index 100% rename from examples/online_serving/disaggregated_serving/README.md rename to examples/disaggregated/disaggregated_serving/README.md diff --git a/examples/online_serving/disaggregated_serving/disagg_proxy_demo.py b/examples/disaggregated/disaggregated_serving/disagg_proxy_demo.py similarity index 99% rename from examples/online_serving/disaggregated_serving/disagg_proxy_demo.py rename to examples/disaggregated/disaggregated_serving/disagg_proxy_demo.py index 763361a30e..57deef6a15 100644 --- a/examples/online_serving/disaggregated_serving/disagg_proxy_demo.py +++ b/examples/disaggregated/disaggregated_serving/disagg_proxy_demo.py @@ -5,7 +5,7 @@ This file provides a disaggregated prefilling proxy demo to demonstrate an example usage of XpYd disaggregated prefilling. We can launch multiple vllm instances (2 for prefill and 2 for decode), and launch this proxy demo through: - python3 examples/online_serving/disaggregated_serving/disagg_proxy_demo.py \ + python3 examples/disaggregated/disaggregated_serving/disagg_proxy_demo.py \ --model $model_name \ --prefill localhost:8100 localhost:8101 \ --decode localhost:8200 localhost:8201 \ diff --git a/examples/online_serving/disaggregated_serving/example_mm_serve.py b/examples/disaggregated/disaggregated_serving/example_mm_serve.py similarity index 100% rename from examples/online_serving/disaggregated_serving/example_mm_serve.py rename to examples/disaggregated/disaggregated_serving/example_mm_serve.py diff --git a/examples/online_serving/disaggregated_serving/kv_events.sh b/examples/disaggregated/disaggregated_serving/kv_events.sh similarity index 100% rename from examples/online_serving/disaggregated_serving/kv_events.sh rename to examples/disaggregated/disaggregated_serving/kv_events.sh diff --git a/examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py b/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py similarity index 100% rename from examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py rename to examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py diff --git a/examples/online_serving/ec_both_encoder/ec_both_encoder.sh b/examples/disaggregated/ec_both_encoder/ec_both_encoder.sh old mode 100755 new mode 100644 similarity index 100% rename from examples/online_serving/ec_both_encoder/ec_both_encoder.sh rename to examples/disaggregated/ec_both_encoder/ec_both_encoder.sh diff --git a/examples/offline_inference/disaggregated-prefill-v1/README.md b/examples/disaggregated/example_connector/README.md similarity index 81% rename from examples/offline_inference/disaggregated-prefill-v1/README.md rename to examples/disaggregated/example_connector/README.md index abf6883f8d..43f1622389 100644 --- a/examples/offline_inference/disaggregated-prefill-v1/README.md +++ b/examples/disaggregated/example_connector/README.md @@ -5,6 +5,6 @@ This example contains scripts that demonstrate disaggregated prefill in the offl ## Files - `run.sh` - A helper script that will run `prefill_example.py` and `decode_example.py` sequentially. - - Make sure you are in the `examples/offline_inference/disaggregated-prefill-v1` directory before running `run.sh`. + - Make sure you are in the `examples/disaggregated/example_connector` directory before running `run.sh`. - `prefill_example.py` - A script which performs prefill only, saving the KV state to the `local_storage` directory and the prompts to `output.txt`. - `decode_example.py` - A script which performs decode only, loading the KV state from the `local_storage` directory and the prompts from `output.txt`. diff --git a/examples/offline_inference/disaggregated-prefill-v1/decode_example.py b/examples/disaggregated/example_connector/decode_example.py similarity index 100% rename from examples/offline_inference/disaggregated-prefill-v1/decode_example.py rename to examples/disaggregated/example_connector/decode_example.py diff --git a/examples/offline_inference/disaggregated-prefill-v1/prefill_example.py b/examples/disaggregated/example_connector/prefill_example.py similarity index 100% rename from examples/offline_inference/disaggregated-prefill-v1/prefill_example.py rename to examples/disaggregated/example_connector/prefill_example.py diff --git a/examples/offline_inference/disaggregated-prefill-v1/run.sh b/examples/disaggregated/example_connector/run.sh similarity index 100% rename from examples/offline_inference/disaggregated-prefill-v1/run.sh rename to examples/disaggregated/example_connector/run.sh diff --git a/examples/offline_inference/prefix_caching_flexkv.py b/examples/disaggregated/flexkv_connector/prefix_caching_flexkv.py similarity index 98% rename from examples/offline_inference/prefix_caching_flexkv.py rename to examples/disaggregated/flexkv_connector/prefix_caching_flexkv.py index f2ffb75ef8..b67c245931 100644 --- a/examples/offline_inference/prefix_caching_flexkv.py +++ b/examples/disaggregated/flexkv_connector/prefix_caching_flexkv.py @@ -14,7 +14,7 @@ Requirements: Usage: 1. Run this script: - python examples/offline_inference/prefix_caching_flexkv.py \ + python examples/disaggregated/flexkv_connector/prefix_caching_flexkv.py \ --model /path/to/your/model 2. Arguments: diff --git a/examples/offline_inference/kv_load_failure_recovery/README.md b/examples/disaggregated/kv_load_failure_recovery_offline/README.md similarity index 91% rename from examples/offline_inference/kv_load_failure_recovery/README.md rename to examples/disaggregated/kv_load_failure_recovery_offline/README.md index 176141b5de..7205b2135e 100644 --- a/examples/offline_inference/kv_load_failure_recovery/README.md +++ b/examples/disaggregated/kv_load_failure_recovery_offline/README.md @@ -1,12 +1,12 @@ # KV Load Failure Recovery Test -This example builds upon the `disaggregated-prefill-v1` example in `examples/offline_inference`. +This example builds upon the `example_connector` example in `examples/disaggregated`. It demonstrates vLLM's ability to recover from KV load failures in both synchronous and asynchronous loading modes. The goal is to verify that vLLM correctly identifies invalid KV blocks, reschedules the affected requests, and ensures successful and consistent output. ## Files -- `prefill_example.py` – performs the prefill stage and saves KV data (same as in `disaggregated-prefill-v1`). +- `prefill_example.py` – performs the prefill stage and saves KV data (same as in `example_connector`). - `decode_example.py` – performs the decode stage. Accepts: - `--simulate-failure`: simulates KV load failure using a custom connector. - `--async-load`: enables asynchronous KV loading mode. diff --git a/examples/offline_inference/kv_load_failure_recovery/decode_example.py b/examples/disaggregated/kv_load_failure_recovery_offline/decode_example.py similarity index 100% rename from examples/offline_inference/kv_load_failure_recovery/decode_example.py rename to examples/disaggregated/kv_load_failure_recovery_offline/decode_example.py diff --git a/examples/offline_inference/kv_load_failure_recovery/load_recovery_example_connector.py b/examples/disaggregated/kv_load_failure_recovery_offline/load_recovery_example_connector.py similarity index 100% rename from examples/offline_inference/kv_load_failure_recovery/load_recovery_example_connector.py rename to examples/disaggregated/kv_load_failure_recovery_offline/load_recovery_example_connector.py diff --git a/examples/offline_inference/kv_load_failure_recovery/prefill_example.py b/examples/disaggregated/kv_load_failure_recovery_offline/prefill_example.py similarity index 100% rename from examples/offline_inference/kv_load_failure_recovery/prefill_example.py rename to examples/disaggregated/kv_load_failure_recovery_offline/prefill_example.py diff --git a/examples/offline_inference/kv_load_failure_recovery/run.sh b/examples/disaggregated/kv_load_failure_recovery_offline/run.sh old mode 100755 new mode 100644 similarity index 100% rename from examples/offline_inference/kv_load_failure_recovery/run.sh rename to examples/disaggregated/kv_load_failure_recovery_offline/run.sh diff --git a/examples/others/lmcache/README.md b/examples/disaggregated/lmcache/README.md similarity index 100% rename from examples/others/lmcache/README.md rename to examples/disaggregated/lmcache/README.md diff --git a/examples/others/lmcache/cpu_offload_lmcache.py b/examples/disaggregated/lmcache/cpu_offload_lmcache.py similarity index 100% rename from examples/others/lmcache/cpu_offload_lmcache.py rename to examples/disaggregated/lmcache/cpu_offload_lmcache.py diff --git a/examples/others/lmcache/disagg_prefill_lmcache_v0.py b/examples/disaggregated/lmcache/disagg_prefill_lmcache_v0.py similarity index 100% rename from examples/others/lmcache/disagg_prefill_lmcache_v0.py rename to examples/disaggregated/lmcache/disagg_prefill_lmcache_v0.py diff --git a/examples/others/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-decoder-config.yaml b/examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-decoder-config.yaml similarity index 100% rename from examples/others/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-decoder-config.yaml rename to examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-decoder-config.yaml diff --git a/examples/others/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-prefiller-config.yaml b/examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-prefiller-config.yaml similarity index 100% rename from examples/others/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-prefiller-config.yaml rename to examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/configs/lmcache-prefiller-config.yaml diff --git a/examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh b/examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh similarity index 100% rename from examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh rename to examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh diff --git a/examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py b/examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py similarity index 100% rename from examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py rename to examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py diff --git a/examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_vllm_launcher.sh b/examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_vllm_launcher.sh similarity index 100% rename from examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_vllm_launcher.sh rename to examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_vllm_launcher.sh diff --git a/examples/others/lmcache/kv_cache_sharing_lmcache_v1.py b/examples/disaggregated/lmcache/kv_cache_sharing_lmcache_v1.py similarity index 100% rename from examples/others/lmcache/kv_cache_sharing_lmcache_v1.py rename to examples/disaggregated/lmcache/kv_cache_sharing_lmcache_v1.py diff --git a/examples/online_serving/disaggregated_serving/mooncake_connector/mooncake_connector_proxy.py b/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py similarity index 100% rename from examples/online_serving/disaggregated_serving/mooncake_connector/mooncake_connector_proxy.py rename to examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py diff --git a/examples/online_serving/disaggregated_serving/mooncake_connector/run_mooncake_connector.sh b/examples/disaggregated/mooncake_connector/run_mooncake_connector.sh similarity index 100% rename from examples/online_serving/disaggregated_serving/mooncake_connector/run_mooncake_connector.sh rename to examples/disaggregated/mooncake_connector/run_mooncake_connector.sh diff --git a/examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh b/examples/disaggregated/p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh similarity index 100% rename from examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh rename to examples/disaggregated/p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh diff --git a/examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_proxy_p2p_nccl_xpyd.py b/examples/disaggregated/p2p_nccl_xpyd/disagg_proxy_p2p_nccl_xpyd.py similarity index 100% rename from examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_proxy_p2p_nccl_xpyd.py rename to examples/disaggregated/p2p_nccl_xpyd/disagg_proxy_p2p_nccl_xpyd.py diff --git a/examples/offline_inference/batch_llm_inference.py b/examples/ray_serving/batch_llm_inference.py similarity index 100% rename from examples/offline_inference/batch_llm_inference.py rename to examples/ray_serving/batch_llm_inference.py diff --git a/examples/online_serving/elastic_ep/bench.sh b/examples/ray_serving/elastic_ep/bench.sh similarity index 100% rename from examples/online_serving/elastic_ep/bench.sh rename to examples/ray_serving/elastic_ep/bench.sh diff --git a/examples/online_serving/elastic_ep/scale.py b/examples/ray_serving/elastic_ep/scale.py similarity index 100% rename from examples/online_serving/elastic_ep/scale.py rename to examples/ray_serving/elastic_ep/scale.py diff --git a/examples/online_serving/elastic_ep/serve_deepseek_v2.sh b/examples/ray_serving/elastic_ep/serve_deepseek_v2.sh similarity index 100% rename from examples/online_serving/elastic_ep/serve_deepseek_v2.sh rename to examples/ray_serving/elastic_ep/serve_deepseek_v2.sh diff --git a/examples/online_serving/multi-node-serving.sh b/examples/ray_serving/multi-node-serving.sh similarity index 100% rename from examples/online_serving/multi-node-serving.sh rename to examples/ray_serving/multi-node-serving.sh diff --git a/examples/online_serving/ray_serve_deepseek.py b/examples/ray_serving/ray_serve_deepseek.py similarity index 100% rename from examples/online_serving/ray_serve_deepseek.py rename to examples/ray_serving/ray_serve_deepseek.py diff --git a/examples/online_serving/run_cluster.sh b/examples/ray_serving/run_cluster.sh similarity index 100% rename from examples/online_serving/run_cluster.sh rename to examples/ray_serving/run_cluster.sh diff --git a/tests/v1/ec_connector/integration/README.md b/tests/v1/ec_connector/integration/README.md index 2dbcb307fd..2e122680ca 100644 --- a/tests/v1/ec_connector/integration/README.md +++ b/tests/v1/ec_connector/integration/README.md @@ -122,7 +122,7 @@ Quick sanity check: - Encoder cache should enable exact output reproduction - Test cleans up all instances and cache files after completion - Safe to run multiple times (idempotent) -- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/online_serving/disaggregated_encoder/README.md` +- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md` ## Requirements diff --git a/tests/v1/ec_connector/integration/run_epd_correctness_test.sh b/tests/v1/ec_connector/integration/run_epd_correctness_test.sh index e199a3ecea..65716444a5 100644 --- a/tests/v1/ec_connector/integration/run_epd_correctness_test.sh +++ b/tests/v1/ec_connector/integration/run_epd_correctness_test.sh @@ -185,7 +185,7 @@ run_epd_1e_1pd() { # Start proxy echo "Starting EPD proxy on port $PROXY_PORT" - python "${GIT_ROOT}/examples/online_serving/disaggregated_encoder/disagg_epd_proxy.py" \ + python "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \ --host "0.0.0.0" \ --port "$PROXY_PORT" \ --encode-servers-urls "http://localhost:$ENCODE_PORT" \ @@ -411,7 +411,7 @@ run_epd_1e_1p_1d() { # Start proxy echo "Starting EPD proxy on port $PROXY_PORT" - python "${GIT_ROOT}/examples/online_serving/disaggregated_encoder/disagg_epd_proxy.py" \ + python "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \ --host "0.0.0.0" \ --port "$PROXY_PORT" \ --encode-servers-urls "http://localhost:$ENCODE_PORT" \ diff --git a/vllm/distributed/kv_transfer/README.md b/vllm/distributed/kv_transfer/README.md index 39377aabcc..64be074cf9 100644 --- a/vllm/distributed/kv_transfer/README.md +++ b/vllm/distributed/kv_transfer/README.md @@ -22,7 +22,7 @@ NOTE: If you want to not only transfer KV caches, but adjust the model execution ## Disaggregated prefilling -The example usage is in [this file](../../../examples/online_serving/disaggregated_prefill.sh). +The example usage is in [this file](../../../examples/disaggregated/disaggregated_prefill.sh). Here is the diagram of how we run disaggregated prefilling.