[TPU][V1] Refine tpu_model_runner to mitigate future recompilation issues (#16275)
Signed-off-by: Chengji Yao <chengjiyao@google.com>
This commit is contained in:
@@ -3,7 +3,6 @@ from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch_xla.core.xla_model as xm
|
||||
|
||||
from vllm.v1.worker.gpu_input_batch import InputBatch
|
||||
|
||||
@@ -24,15 +23,15 @@ class TPUSupportedSamplingMetadata:
|
||||
# This class exposes a more xla-friendly interface than SamplingMetadata
|
||||
# on TPU, in particular all arguments should be traceable and no optionals
|
||||
# are allowed, to avoid graph recompilation on Nones.
|
||||
temperature: torch.Tensor
|
||||
temperature: torch.Tensor = None
|
||||
|
||||
min_p: torch.Tensor
|
||||
min_p: torch.Tensor = None
|
||||
# Still too slow on forward_native!
|
||||
top_k: torch.Tensor = None
|
||||
top_p: torch.Tensor = None
|
||||
|
||||
# Greedy sampling flag for compiling single xla graph.
|
||||
all_greedy: torch.Tensor = None
|
||||
all_greedy: bool = True
|
||||
|
||||
# Generator not supported by xla
|
||||
generators: dict[int,
|
||||
@@ -57,64 +56,58 @@ class TPUSupportedSamplingMetadata:
|
||||
|
||||
allowed_token_ids_mask = None
|
||||
bad_words_token_ids = None
|
||||
indices_do_sample: torch.Tensor = None
|
||||
|
||||
@classmethod
|
||||
def from_input_batch(
|
||||
cls, input_batch: InputBatch,
|
||||
indices_do_sample: torch.Tensor) -> "TPUSupportedSamplingMetadata":
|
||||
cls,
|
||||
input_batch: InputBatch,
|
||||
padded_num_reqs: int,
|
||||
xla_device: torch.device,
|
||||
generate_params_if_all_greedy: bool = False
|
||||
) -> "TPUSupportedSamplingMetadata":
|
||||
"""
|
||||
Copy sampling tensors slices from `input_batch` to on device tensors.
|
||||
|
||||
`InputBatch._make_sampling_metadata` causes recompilation on XLA as it
|
||||
slices dynamic shapes on device tensors. This impl moves the dynamic
|
||||
ops to CPU and produces tensors of fixed `padded_num_reqs` size. It
|
||||
also reuses the on-device persistent tensors managed in `input_batch`
|
||||
to reduce waste.
|
||||
ops to CPU and produces tensors of fixed `padded_num_reqs` size.
|
||||
|
||||
`indices_do_sample` contains the indices to be fed to the Sampler,
|
||||
normally one per request, here padded to the closest pre-compiled shape
|
||||
We expect sampling params tensors to be padded to the same fixed shape.
|
||||
|
||||
Eg. 3 requests, tensors padded to 4
|
||||
temperature: [0.7, 0.2, 0.9]=>[0.7, 0.2, 0.9, 0.0]
|
||||
sample indices: [4, 10, 11]=>indices_do_sample: [4, 10, 11, 0]
|
||||
Args:
|
||||
input_batch: The input batch containing sampling parameters.
|
||||
padded_num_reqs: The padded number of requests.
|
||||
xla_device: The XLA device.
|
||||
generate_params_if_all_greedy: If True, generate sampling parameters
|
||||
even if all requests are greedy. this is useful for cases where
|
||||
we want to pre-compile a graph with sampling parameters, even if
|
||||
they are not strictly needed for greedy decoding.
|
||||
"""
|
||||
num_reqs = input_batch.num_reqs
|
||||
padded_num_reqs = len(indices_do_sample)
|
||||
# Early return to avoid unnecessary cpu to tpu copy
|
||||
if (input_batch.all_greedy is True
|
||||
and generate_params_if_all_greedy is False):
|
||||
return cls(all_greedy=True)
|
||||
|
||||
def copy_slice(cpu_tensor: torch.Tensor, tpu_tensor: torch.Tensor,
|
||||
fill_val) -> torch.Tensor:
|
||||
# Copy slice from CPU to corresponding TPU pre-allocated tensor.
|
||||
num_reqs = input_batch.num_reqs
|
||||
|
||||
def fill_slice(cpu_tensor: torch.Tensor, fill_val) -> torch.Tensor:
|
||||
# Pad value is the default one.
|
||||
cpu_tensor[num_reqs:padded_num_reqs] = fill_val
|
||||
# Subtle compilation: len(tpu_tensor) must be >= `padded_num_reqs`
|
||||
tpu_tensor[:padded_num_reqs] = cpu_tensor[:padded_num_reqs]
|
||||
|
||||
# NOTE NickLucche The sync CPU-TPU graph we produce here must be
|
||||
# consistent. We can't have flags to skip copies or we'll end up
|
||||
# recompiling.
|
||||
copy_slice(input_batch.temperature_cpu_tensor, input_batch.temperature,
|
||||
fill_slice(input_batch.temperature_cpu_tensor,
|
||||
DEFAULT_SAMPLING_PARAMS["temperature"])
|
||||
# TODO Temporarily disabled until sampling options are enabled
|
||||
# copy_slice(input_batch.top_p_cpu_tensor, input_batch.top_p)
|
||||
# copy_slice(input_batch.top_k_cpu_tensor, input_batch.top_k)
|
||||
copy_slice(input_batch.min_p_cpu_tensor, input_batch.min_p,
|
||||
# fill_slice(input_batch.top_p_cpu_tensor)
|
||||
# fill_slice(input_batch.top_k_cpu_tensor)
|
||||
fill_slice(input_batch.min_p_cpu_tensor,
|
||||
DEFAULT_SAMPLING_PARAMS["min_p"])
|
||||
|
||||
xm.mark_step()
|
||||
xm.wait_device_ops()
|
||||
|
||||
# Slice persistent device tensors to a fixed pre-compiled padded shape.
|
||||
return cls(
|
||||
temperature=input_batch.temperature[:padded_num_reqs],
|
||||
# Scalar tensor for xla-friendly tracing.
|
||||
all_greedy=torch.tensor(input_batch.all_greedy,
|
||||
dtype=torch.bool,
|
||||
device=input_batch.device),
|
||||
temperature=input_batch.temperature_cpu_tensor[:padded_num_reqs].
|
||||
to(xla_device),
|
||||
all_greedy=input_batch.all_greedy,
|
||||
# TODO enable more and avoid returning None values
|
||||
top_p=None, # input_batch.top_p[:padded_num_reqs],
|
||||
top_k=None, # input_batch.top_k[:padded_num_reqs],
|
||||
min_p=input_batch.min_p[:padded_num_reqs],
|
||||
generators=input_batch.generators,
|
||||
indices_do_sample=indices_do_sample)
|
||||
min_p=input_batch.min_p_cpu_tensor[:padded_num_reqs].to(
|
||||
xla_device),
|
||||
generators=input_batch.generators)
|
||||
|
||||
Reference in New Issue
Block a user