[Hardware][TPU] Initial support of model parallelism with single worker using SPMD (#18011)
Signed-off-by: Siyuan Liu <lsiyuan@google.com> Co-authored-by: Hossein Sarshar <hossein.sarshar@gmail.com> Co-authored-by: Chengji Yao <chengjiyao@google.com>
This commit is contained in:
co-authored by
Hossein Sarshar
Chengji Yao
parent
c57d577e8d
commit
9112b443a0
@@ -1,5 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
prompts = [
|
||||
@@ -18,14 +21,28 @@ sampling_params = SamplingParams(temperature=0, top_p=1.0, n=N, max_tokens=16)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="TPU offline inference example")
|
||||
parser.add_argument("--use-spmd", action="store_true", help="Enable SPMD mode")
|
||||
args = parser.parse_args()
|
||||
|
||||
llm_args = {
|
||||
"model": "Qwen/Qwen2-1.5B-Instruct",
|
||||
"max_num_batched_tokens": 64,
|
||||
"max_num_seqs": 4,
|
||||
"max_model_len": 128,
|
||||
}
|
||||
if args.use_spmd:
|
||||
os.environ["VLLM_XLA_USE_SPMD"] = "1"
|
||||
# Can only hardcode the number of chips for now.
|
||||
# calling xr.global_runtime_device_count() beforeing init SPMD env in
|
||||
# torch_xla will mess up the distributed env.
|
||||
llm_args["tensor_parallel_size"] = 8
|
||||
# Use Llama, for num_kv_heads = 8.
|
||||
llm_args["model"] = "meta-llama/Llama-3.1-8B-Instruct"
|
||||
|
||||
# Set `enforce_eager=True` to avoid ahead-of-time compilation.
|
||||
# In real workloads, `enforace_eager` should be `False`.
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen2-1.5B-Instruct",
|
||||
max_num_batched_tokens=64,
|
||||
max_num_seqs=4,
|
||||
max_model_len=128,
|
||||
)
|
||||
llm = LLM(**llm_args)
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
print("-" * 50)
|
||||
for output, answer in zip(outputs, answers):
|
||||
|
||||
Reference in New Issue
Block a user