From 3d300aecb1e6639872b698bd74ed38fb81d9603e Mon Sep 17 00:00:00 2001 From: Peter Pan Date: Thu, 11 Jun 2026 02:17:11 +0800 Subject: [PATCH] [Doc] Switch K8S examples to default MP mode (#39400) Signed-off-by: Peter Pan Signed-off-by: Peter Pan Signed-off-by: Kyle Sayers Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Kyle Sayers Co-authored-by: Flora Feng <4florafeng@gmail.com> --- docs/deployment/frameworks/lws.md | 326 +++++++++++++------ docs/deployment/integrations/kthena.md | 358 ++++++++++++++------- examples/ray_serving/multi-node-serving.sh | 2 +- 3 files changed, 467 insertions(+), 219 deletions(-) diff --git a/docs/deployment/frameworks/lws.md b/docs/deployment/frameworks/lws.md index 47586bcd70..5aae73c8a3 100644 --- a/docs/deployment/frameworks/lws.md +++ b/docs/deployment/frameworks/lws.md @@ -7,108 +7,202 @@ vLLM can be deployed with [LWS](https://github.com/kubernetes-sigs/lws) on Kuber ## Prerequisites -* At least two Kubernetes nodes, each with 8 GPUs, are required. -* Install LWS by following the instructions found [here](https://lws.sigs.k8s.io/docs/installation/). +- At least two Kubernetes nodes, each with 8 GPUs, are required. +- Install LWS by following the instructions found [here](https://lws.sigs.k8s.io/docs/installation/). ## Deploy and Serve -Deploy the following yaml file `lws.yaml` +Deploy the following yaml file `lws.yaml` (we have examples that use multiprocessing or Ray): -??? code "Yaml" +??? code "lws.yaml" + === "Multiprocessing (default)" + ```yaml + apiVersion: leaderworkerset.x-k8s.io/v1 + kind: LeaderWorkerSet + metadata: + name: vllm + spec: + replicas: 1 + leaderWorkerTemplate: + size: 2 + restartPolicy: RecreateGroupOnPodRestart + leaderTemplate: + metadata: + labels: + role: leader + spec: + containers: + - name: vllm-leader + image: docker.io/vllm/vllm-openai:latest + env: + - name: HF_TOKEN + value: + command: + - sh + - -c + - "vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size $(LWS_GROUP_SIZE) --nnodes $(LWS_GROUP_SIZE) --node-rank $(LWS_WORKER_INDEX) --master-addr $(LWS_LEADER_ADDRESS) --port 8080" + resources: + limits: + nvidia.com/gpu: "8" + memory: 1124Gi + ephemeral-storage: 800Gi + requests: + ephemeral-storage: 800Gi + cpu: 125 + ports: + - containerPort: 8080 + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 10 + volumeMounts: + - mountPath: /dev/shm + name: dshm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 15Gi + workerTemplate: + spec: + containers: + - name: vllm-worker + image: docker.io/vllm/vllm-openai:latest + command: + - sh + - -c + - "vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size $(LWS_GROUP_SIZE) --nnodes $(LWS_GROUP_SIZE) --node-rank $(LWS_WORKER_INDEX) --master-addr $(LWS_LEADER_ADDRESS) --headless" + resources: + limits: + nvidia.com/gpu: "8" + memory: 1124Gi + ephemeral-storage: 800Gi + requests: + ephemeral-storage: 800Gi + cpu: 125 + env: + - name: HF_TOKEN + value: + volumeMounts: + - mountPath: /dev/shm + name: dshm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 15Gi + --- + apiVersion: v1 + kind: Service + metadata: + name: vllm-leader + spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + leaderworkerset.sigs.k8s.io/name: vllm + role: leader + type: ClusterIP + ``` - ```yaml - apiVersion: leaderworkerset.x-k8s.io/v1 - kind: LeaderWorkerSet - metadata: - name: vllm - spec: - replicas: 1 - leaderWorkerTemplate: - size: 2 - restartPolicy: RecreateGroupOnPodRestart - leaderTemplate: - metadata: - labels: - role: leader - spec: - containers: - - name: vllm-leader - image: docker.io/vllm/vllm-openai:latest - env: - - name: HF_TOKEN - value: - command: - - sh - - -c - - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=$(LWS_GROUP_SIZE); - vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline_parallel_size 2" - resources: - limits: - nvidia.com/gpu: "8" - memory: 1124Gi - ephemeral-storage: 800Gi - requests: - ephemeral-storage: 800Gi - cpu: 125 - ports: - - containerPort: 8080 - readinessProbe: - tcpSocket: - port: 8080 - initialDelaySeconds: 15 - periodSeconds: 10 - volumeMounts: - - mountPath: /dev/shm - name: dshm - volumes: - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 15Gi - workerTemplate: - spec: - containers: - - name: vllm-worker - image: docker.io/vllm/vllm-openai:latest - command: - - sh - - -c - - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(LWS_LEADER_ADDRESS)" - resources: - limits: - nvidia.com/gpu: "8" - memory: 1124Gi - ephemeral-storage: 800Gi - requests: - ephemeral-storage: 800Gi - cpu: 125 - env: - - name: HF_TOKEN - value: - volumeMounts: - - mountPath: /dev/shm - name: dshm - volumes: - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 15Gi - --- - apiVersion: v1 - kind: Service - metadata: - name: vllm-leader - spec: - ports: - - name: http - port: 8080 - protocol: TCP - targetPort: 8080 - selector: - leaderworkerset.sigs.k8s.io/name: vllm - role: leader - type: ClusterIP - ``` + === "Ray" + ```yaml + apiVersion: leaderworkerset.x-k8s.io/v1 + kind: LeaderWorkerSet + metadata: + name: vllm + spec: + replicas: 1 + leaderWorkerTemplate: + size: 2 + restartPolicy: RecreateGroupOnPodRestart + leaderTemplate: + metadata: + labels: + role: leader + spec: + containers: + - name: vllm-leader + image: docker.io/vllm/vllm-openai:latest + env: + - name: HF_TOKEN + value: + command: + - sh + - -c + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=$(LWS_GROUP_SIZE); + vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline-parallel-size 2 --distributed-executor-backend ray" + resources: + limits: + nvidia.com/gpu: "8" + memory: 1124Gi + ephemeral-storage: 800Gi + requests: + ephemeral-storage: 800Gi + cpu: 125 + ports: + - containerPort: 8080 + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 10 + volumeMounts: + - mountPath: /dev/shm + name: dshm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 15Gi + workerTemplate: + spec: + containers: + - name: vllm-worker + image: docker.io/vllm/vllm-openai:latest + command: + - sh + - -c + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(LWS_LEADER_ADDRESS)" + resources: + limits: + nvidia.com/gpu: "8" + memory: 1124Gi + ephemeral-storage: 800Gi + requests: + ephemeral-storage: 800Gi + cpu: 125 + env: + - name: HF_TOKEN + value: + volumeMounts: + - mountPath: /dev/shm + name: dshm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 15Gi + --- + apiVersion: v1 + kind: Service + metadata: + name: vllm-leader + spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + leaderworkerset.sigs.k8s.io/name: vllm + role: leader + type: ClusterIP + ``` ```bash kubectl apply -f lws.yaml @@ -130,16 +224,37 @@ vllm-0-1 1/1 Running 0 2s Verify that the distributed tensor-parallel inference works: -```bash -kubectl logs vllm-0 |grep -i "Loading model weights took" -``` +=== "Multiprocessing (default)" + ```bash + kubectl logs vllm-0 | grep -i "Model loading" + kubectl logs vllm-0-1 | grep -i "Model loading" + ``` -Should get something similar to this: + Should get something similar to this: -```text -INFO 05-08 03:20:24 model_runner.py:173] Loading model weights took 0.1189 GB -(RayWorkerWrapper pid=169, ip=10.20.0.197) INFO 05-08 03:20:28 model_runner.py:173] Loading model weights took 0.1189 GB -``` + POD 0 (PP Rank 0) + + ```text + (Worker_PP0_TP0 pid=601) INFO 04-28 08:16:58 [gpu_model_runner.py:4820] Model loading took 3.82 GiB memory and 157.996399 seconds + ``` + + POD 1 (PP Rank 1) + + ```text + (Worker_PP1_TP0 pid=396) INFO 04-28 08:17:09 [gpu_model_runner.py:4820] Model loading took 3.82 GiB memory and 168.878781 seconds + ``` + +=== "Ray" + ```bash + kubectl logs vllm-0 | grep -i "Loading model weights took" + ``` + + Should get something similar to this: + + ```text + INFO 05-08 03:20:24 model_runner.py:173] Loading model weights took 0.1189 GB + (RayWorkerWrapper pid=169, ip=10.20.0.197) INFO 05-08 03:20:28 model_runner.py:173] Loading model weights took 0.1189 GB + ``` ## Access ClusterIP service @@ -173,7 +288,6 @@ curl http://localhost:8080/v1/completions \ The output should be similar to the following ??? console "Output" - ```text { "id": "cmpl-1bb34faba88b43f9862cfbfb2200949d", diff --git a/docs/deployment/integrations/kthena.md b/docs/deployment/integrations/kthena.md index 03ef190e55..7cc3f14a71 100644 --- a/docs/deployment/integrations/kthena.md +++ b/docs/deployment/integrations/kthena.md @@ -64,36 +64,74 @@ A simplified version of the example (`llama-multinode`) looks like: - `spec.replicas: 1` – one `ServingGroup` (one logical model deployment). - `roles`: - `entryTemplate` – defines **leader** pods that run: - - vLLM’s **multi-node cluster bootstrap script** (Ray cluster). + - vLLM’s **multi-node cluster bootstrap script**. - vLLM **OpenAI-compatible API server**. - - `workerTemplate` – defines **worker** pods that join the leader’s Ray cluster. + - `workerTemplate` – defines **worker** pods to join the leader’s Ray cluster (Ray backend) or to join same distributed process group (multiprocessing backend). Key points from the example YAML: -- **Image**: `vllm/vllm-openai:latest` (matches upstream vLLM images). -- **Command** (leader): +Image: `vllm/vllm-openai:latest` (matches upstream vLLM images). +Commands: - ```yaml - command: - - sh - - -c - - > - bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=2; - vllm serve meta-llama/Llama-3.1-405B-Instruct - --port 8080 - --tensor-parallel-size 8 - --pipeline-parallel-size 2 - ``` +??? code "Yaml" + === "Multiprocessing (default)" + Leader: -- **Command** (worker): + ```yaml + command: + - sh + - -c + - > + vllm serve meta-llama/Llama-3.1-405B-Instruct + --tensor-parallel-size 8 + --pipeline-parallel-size 2 + --nnodes=2 + --node-rank=0 + --master-addr=$(ENTRY_ADDRESS) + --port 8080 + ``` - ```yaml - command: - - sh - - -c - - > - bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS) - ``` + Worker: + + ```yaml + command: + - sh + - -c + - > + vllm serve meta-llama/Llama-3.1-405B-Instruct + --tensor-parallel-size 8 + --pipeline-parallel-size 2 + --nnodes=2 + --node-rank=1 + --master-addr=$(ENTRY_ADDRESS) + --headless + ``` + + === "Ray" + Leader: + + ```yaml + command: + - sh + - -c + - > + bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh + leader --ray_cluster_size=2; python3 -m + vllm.entrypoints.openai.api_server --port 8080 --model + meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8 + --pipeline-parallel-size 2 + ``` + + Worker: + + ```yaml + command: + - sh + - -c + - > + bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh + worker --ray_address=$(ENTRY_ADDRESS) + ``` --- @@ -111,96 +149,192 @@ kubectl create secret generic hf-token \ ### 3.2 Apply the `ModelServing` +Save one of the following manifests to `modelserving.yaml`: + +??? code "modelserving.yaml" + === "Multiprocessing (default)" + ```yaml + apiVersion: workload.serving.volcano.sh/v1alpha1 + kind: ModelServing + metadata: + name: llama-multinode + namespace: default + spec: + schedulerName: volcano + replicas: 1 # group replicas + template: + restartGracePeriodSeconds: 60 + gangPolicy: + minRoleReplicas: + 405b: 1 + roles: + - name: 405b + replicas: 2 + entryTemplate: + spec: + containers: + - name: leader + image: vllm/vllm-openai:latest + env: + - name: HUGGING_FACE_HUB_TOKEN + valueFrom: + secretKeyRef: + name: hf-token + key: HUGGING_FACE_HUB_TOKEN + command: + - sh + - -c + - "vllm serve meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size 2 --nnodes 2 --node-rank 0 --master-addr $(ENTRY_ADDRESS) --distributed-executor-backend mp --port 8080" + resources: + limits: + nvidia.com/gpu: "8" + memory: 1124Gi + ephemeral-storage: 800Gi + requests: + ephemeral-storage: 800Gi + cpu: 125 + ports: + - containerPort: 8080 + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 10 + volumeMounts: + - mountPath: /dev/shm + name: dshm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 15Gi + workerReplicas: 1 + workerTemplate: + spec: + containers: + - name: worker + image: vllm/vllm-openai:latest + command: + - sh + - -c + - "vllm serve meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size 2 --nnodes 2 --node-rank 1 --master-addr $(ENTRY_ADDRESS) --distributed-executor-backend mp --headless" + resources: + limits: + nvidia.com/gpu: "8" + memory: 1124Gi + ephemeral-storage: 800Gi + requests: + ephemeral-storage: 800Gi + cpu: 125 + env: + - name: HUGGING_FACE_HUB_TOKEN + valueFrom: + secretKeyRef: + name: hf-token + key: HUGGING_FACE_HUB_TOKEN + volumeMounts: + - mountPath: /dev/shm + name: dshm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 15Gi + ``` + + === "Ray" + ```yaml + apiVersion: workload.serving.volcano.sh/v1alpha1 + kind: ModelServing + metadata: + name: llama-multinode + namespace: default + spec: + schedulerName: volcano + replicas: 1 # group replicas + template: + restartGracePeriodSeconds: 60 + gangPolicy: + minRoleReplicas: + 405b: 1 + roles: + - name: 405b + replicas: 2 + entryTemplate: + spec: + containers: + - name: leader + image: vllm/vllm-openai:latest + env: + - name: HUGGING_FACE_HUB_TOKEN + valueFrom: + secretKeyRef: + name: hf-token + key: HUGGING_FACE_HUB_TOKEN + command: + - sh + - -c + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=2; + vllm serve meta-llama/Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline-parallel-size 2" + resources: + limits: + nvidia.com/gpu: "8" + memory: 1124Gi + ephemeral-storage: 800Gi + requests: + ephemeral-storage: 800Gi + cpu: 125 + ports: + - containerPort: 8080 + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 10 + volumeMounts: + - mountPath: /dev/shm + name: dshm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 15Gi + workerReplicas: 1 + workerTemplate: + spec: + containers: + - name: worker + image: vllm/vllm-openai:latest + command: + - sh + - -c + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS)" + resources: + limits: + nvidia.com/gpu: "8" + memory: 1124Gi + ephemeral-storage: 800Gi + requests: + ephemeral-storage: 800Gi + cpu: 125 + env: + - name: HUGGING_FACE_HUB_TOKEN + valueFrom: + secretKeyRef: + name: hf-token + key: HUGGING_FACE_HUB_TOKEN + volumeMounts: + - mountPath: /dev/shm + name: dshm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 15Gi + ``` + ```bash -cat < [] && \ -# vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline_parallel_size 2 +# vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline-parallel-size 2 --distributed-executor-backend ray # # On each worker node, start the Ray worker node process. # ./multi-node-serving.sh worker --ray_address= --ray_port=6379 []