[Perf] Avoid h2d copies from non-pinned CPU tensors (#54299)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -11,6 +11,7 @@ from vllm.config import get_current_vllm_config
|
||||
from vllm.config.pooler import TokenPoolingType
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
TokenPoolingMethodOutputItem: TypeAlias = torch.Tensor | None
|
||||
@@ -131,7 +132,7 @@ class StepPool(AllPool):
|
||||
|
||||
if step_tag_id is not None:
|
||||
idx_cpu = (token_id_cpu == step_tag_id).nonzero(as_tuple=True)[0]
|
||||
idx = idx_cpu.to(data.device, non_blocking=True)
|
||||
idx = async_tensor_h2d(idx_cpu, data.device)
|
||||
data = data[idx]
|
||||
|
||||
pooled_data.append(data)
|
||||
|
||||
@@ -158,13 +158,13 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
|
||||
yield module
|
||||
return
|
||||
|
||||
original_device_states: dict[str, torch.device] = {}
|
||||
cpu_params: set[str] = set()
|
||||
uva_offloaded_parameters: list[str] = []
|
||||
|
||||
# Store original device states and move parameters to GPU if they're on CPU
|
||||
# Store which parameters are on CPU and move them to the GPU
|
||||
for name, p in module.named_parameters():
|
||||
if p.device.type == "cpu":
|
||||
original_device_states[name] = p.device
|
||||
cpu_params.add(name)
|
||||
p.data = p.data.to(target_device)
|
||||
if getattr(p, "_vllm_is_uva_offloaded", False):
|
||||
uva_offloaded_parameters.append(name)
|
||||
@@ -178,20 +178,21 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
|
||||
is_pin_memory_available()
|
||||
and not envs.VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY
|
||||
)
|
||||
# Restore parameters to their original devices, ignoring new parameters
|
||||
# Restore the CPU-resident parameters, ignoring new parameters.
|
||||
for name, p in module.named_parameters():
|
||||
if name in original_device_states:
|
||||
original_device: torch.device = original_device_states[name]
|
||||
p.data = p.data.to(original_device)
|
||||
if name in cpu_params:
|
||||
p.data = torch.empty_like(
|
||||
p.data, device="cpu", pin_memory=use_pin_memory
|
||||
).copy_(p.data)
|
||||
|
||||
# parameter is UVA offloaded, but was replaced with a new device tensor
|
||||
# re-offload it to CPU using UVA
|
||||
if name in uva_offloaded_parameters and not getattr(
|
||||
p, "_vllm_is_uva_offloaded", False
|
||||
):
|
||||
cpu_data = p.data.to(device="cpu")
|
||||
if use_pin_memory:
|
||||
cpu_data = cpu_data.pin_memory()
|
||||
cpu_data = torch.empty_like(
|
||||
p.data, device="cpu", pin_memory=use_pin_memory
|
||||
).copy_(p.data)
|
||||
p.data = get_accelerator_view_from_cpu_tensor(cpu_data)
|
||||
p._vllm_is_uva_offloaded = True
|
||||
|
||||
|
||||
@@ -425,10 +425,11 @@ class Ernie4_5_VisionTransformer(nn.Module):
|
||||
return self.patch_embed.proj.weight.device
|
||||
|
||||
def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
|
||||
device = self.device
|
||||
pos_ids = []
|
||||
for t, h, w in grid_thw:
|
||||
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
|
||||
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
|
||||
hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w)
|
||||
wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1)
|
||||
hpos_ids = (
|
||||
hpos_ids.reshape(
|
||||
h // self.spatial_merge_size,
|
||||
@@ -453,9 +454,6 @@ class Ernie4_5_VisionTransformer(nn.Module):
|
||||
pos_ids = torch.cat(pos_ids, dim=0)
|
||||
max_grid_size = grid_thw[:, 1:].max()
|
||||
rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
|
||||
# `pos_ids` is built on the host; stage it over non-blocking so the
|
||||
# gather below doesn't index a device tensor with a CPU one.
|
||||
pos_ids = pos_ids.to(rotary_pos_emb_full.device, non_blocking=True)
|
||||
rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
|
||||
return rotary_pos_emb
|
||||
|
||||
|
||||
@@ -709,10 +709,11 @@ class Glm4vVisionTransformer(nn.Module):
|
||||
def rot_pos_emb(
|
||||
self, grid_thw: list[list[int]]
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
device = self.device
|
||||
pos_ids = []
|
||||
for t, h, w in grid_thw:
|
||||
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
|
||||
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
|
||||
hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w)
|
||||
wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1)
|
||||
hpos_ids = (
|
||||
hpos_ids.reshape(
|
||||
h // self.spatial_merge_size,
|
||||
@@ -740,7 +741,6 @@ class Glm4vVisionTransformer(nn.Module):
|
||||
# Use pre-computed cos_sin_cache from RotaryEmbedding
|
||||
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
|
||||
|
||||
pos_ids = pos_ids.to(cos.device, non_blocking=True)
|
||||
cos_combined = cos[pos_ids].flatten(1)
|
||||
sin_combined = sin[pos_ids].flatten(1)
|
||||
return cos_combined, sin_combined, pos_ids
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
from .utils import AutoWeightsLoader, WeightsMapper
|
||||
from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
|
||||
@@ -85,7 +86,9 @@ class Idefics2VisionEmbeddings(nn.Module):
|
||||
1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side
|
||||
)
|
||||
position_ids = torch.full(
|
||||
size=(batch_size, max_nb_patches_h * max_nb_patches_w), fill_value=0
|
||||
size=(batch_size, max_nb_patches_h * max_nb_patches_w),
|
||||
fill_value=0,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
|
||||
with gpu_sync_allowed():
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.model_executor.models.phi4mm_utils import (
|
||||
get_offset,
|
||||
unfold_tensor,
|
||||
)
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
class ConformerEncoderLayer(nn.Module):
|
||||
@@ -586,9 +587,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
|
||||
seq_len, batch_size, self.chunk_size, self.left_chunk
|
||||
)
|
||||
device = xs_pad.device
|
||||
enc_streaming_mask = enc_streaming_mask.contiguous().to(
|
||||
device, non_blocking=True
|
||||
)
|
||||
enc_streaming_mask = async_tensor_h2d(enc_streaming_mask.contiguous(), device)
|
||||
xs_pad = xs_pad.to(device)
|
||||
|
||||
input_tensor = xs_pad
|
||||
@@ -921,9 +920,7 @@ class ConformerEncoder(TransformerEncoderBase):
|
||||
enc_streaming_mask = self._streaming_mask(
|
||||
max_audio_length, batch_size, self.chunk_size, self.left_chunk
|
||||
)
|
||||
enc_streaming_mask = enc_streaming_mask.contiguous().to(
|
||||
device, non_blocking=True
|
||||
)
|
||||
enc_streaming_mask = async_tensor_h2d(enc_streaming_mask.contiguous(), device)
|
||||
if mask is None:
|
||||
return enc_streaming_mask
|
||||
|
||||
|
||||
@@ -715,8 +715,11 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
return self.patch_embed.proj.weight.device
|
||||
|
||||
def rotary_pos_emb_thw(self, t, h, w):
|
||||
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
|
||||
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
|
||||
# Use pre-computed cos_sin_cache from RotaryEmbedding
|
||||
cos, sin = self.rotary_pos_emb.get_cos_sin(max(h, w))
|
||||
|
||||
hpos_ids = torch.arange(h, device=cos.device).unsqueeze(1).expand(-1, w)
|
||||
wpos_ids = torch.arange(w, device=cos.device).unsqueeze(0).expand(h, -1)
|
||||
hpos_ids = (
|
||||
hpos_ids.reshape(
|
||||
h // self.spatial_merge_size,
|
||||
@@ -738,12 +741,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
.flatten()
|
||||
)
|
||||
pos_ids = torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)
|
||||
max_size = max(h, w)
|
||||
|
||||
# Use pre-computed cos_sin_cache from RotaryEmbedding
|
||||
cos, sin = self.rotary_pos_emb.get_cos_sin(max_size)
|
||||
|
||||
pos_ids = pos_ids.to(cos.device, non_blocking=True)
|
||||
cos_combined = cos[pos_ids].flatten(1)
|
||||
sin_combined = sin[pos_ids].flatten(1)
|
||||
|
||||
|
||||
@@ -609,11 +609,12 @@ class Qwen2VisionTransformer(nn.Module):
|
||||
def rot_pos_emb(
|
||||
self, grid_thw: list[list[int]]
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
device = self.device
|
||||
pos_ids = []
|
||||
max_grid_size = 0
|
||||
for t, h, w in grid_thw:
|
||||
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
|
||||
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
|
||||
hpos_ids = torch.arange(h, device=device).unsqueeze(1).expand(-1, w)
|
||||
wpos_ids = torch.arange(w, device=device).unsqueeze(0).expand(h, -1)
|
||||
hpos_ids = (
|
||||
hpos_ids.reshape(
|
||||
h // self.spatial_merge_size,
|
||||
@@ -641,7 +642,6 @@ class Qwen2VisionTransformer(nn.Module):
|
||||
# Use pre-computed cos_sin_cache from RotaryEmbedding
|
||||
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
|
||||
|
||||
pos_ids = pos_ids.to(cos.device, non_blocking=True)
|
||||
cos_combined = cos[pos_ids].flatten(1)
|
||||
sin_combined = sin[pos_ids].flatten(1)
|
||||
return cos_combined, sin_combined
|
||||
|
||||
@@ -94,6 +94,7 @@ from vllm.transformers_utils.processors.qwen3_asr import (
|
||||
Qwen3ASRProcessor,
|
||||
)
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_ASR_TEXT_TAG = "<asr_text>"
|
||||
@@ -465,8 +466,8 @@ class Qwen3ASRForConditionalGeneration(
|
||||
input_features = audio_input["input_features"]
|
||||
# audio_feature_lengths is keep_on_cpu; the audio tower derives
|
||||
# device placement from feature_lens, so move it explicitly.
|
||||
audio_feature_lengths = audio_input["audio_feature_lengths"].to(
|
||||
input_features.device, non_blocking=True
|
||||
audio_feature_lengths = async_tensor_h2d(
|
||||
audio_input["audio_feature_lengths"], input_features.device
|
||||
)
|
||||
|
||||
audio_output_lengths = _get_feat_extract_output_lengths(audio_feature_lengths)
|
||||
|
||||
@@ -842,9 +842,14 @@ class Qwen3Omni_VisionTransformer(nn.Module):
|
||||
return self.patch_embed.proj.weight.device
|
||||
|
||||
def rot_pos_emb(self, grid_thw):
|
||||
max_grid_size = grid_thw[:, 1:].max()
|
||||
|
||||
# Use pre-computed cos_sin_cache from RotaryEmbedding
|
||||
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
|
||||
|
||||
pos_ids = []
|
||||
for t, h, w in grid_thw:
|
||||
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
|
||||
hpos_ids = torch.arange(h, device=cos.device).unsqueeze(1).expand(-1, w)
|
||||
hpos_ids = hpos_ids.reshape(
|
||||
h // self.spatial_merge_size,
|
||||
self.spatial_merge_size,
|
||||
@@ -854,7 +859,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
|
||||
hpos_ids = hpos_ids.permute(0, 2, 1, 3)
|
||||
hpos_ids = hpos_ids.flatten()
|
||||
|
||||
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
|
||||
wpos_ids = torch.arange(w, device=cos.device).unsqueeze(0).expand(h, -1)
|
||||
wpos_ids = wpos_ids.reshape(
|
||||
h // self.spatial_merge_size,
|
||||
self.spatial_merge_size,
|
||||
@@ -865,12 +870,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
|
||||
wpos_ids = wpos_ids.flatten()
|
||||
pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
|
||||
pos_ids = torch.cat(pos_ids, dim=0)
|
||||
max_grid_size = grid_thw[:, 1:].max()
|
||||
|
||||
# Use pre-computed cos_sin_cache from RotaryEmbedding
|
||||
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
|
||||
|
||||
pos_ids = pos_ids.to(cos.device, non_blocking=True)
|
||||
cos_combined = cos[pos_ids].flatten(1)
|
||||
sin_combined = sin[pos_ids].flatten(1)
|
||||
|
||||
|
||||
@@ -56,6 +56,7 @@ from vllm.multimodal.processing import (
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .clip import dual_encoder_has_text_tokens, merge_dual_encoder_text_and_vision
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
|
||||
@@ -1010,7 +1011,7 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
|
||||
flip_indices_cpu = (
|
||||
starts_cpu[sequence_ids_cpu] + ends_cpu[sequence_ids_cpu]
|
||||
) - (1 + current_positions_cpu)
|
||||
flip_indices = flip_indices_cpu.to(features.device, non_blocking=True)
|
||||
flip_indices = async_tensor_h2d(flip_indices_cpu, features.device)
|
||||
|
||||
return features[flip_indices]
|
||||
|
||||
|
||||
@@ -917,16 +917,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
arch,
|
||||
)
|
||||
# Preparing persistent buffers
|
||||
# Since we do not have explicit synchronization in ModelRunnerV2, we do not pin
|
||||
# reused CPU buffers to avoid a race condition between step N async copies to
|
||||
# GPU and step N+1 buffer updates.
|
||||
self.pin_memory = not vllm_config.use_v2_model_runner and PIN_MEMORY
|
||||
self.paged_kv_indptr = self._make_buffer(max_num_reqs + 1)
|
||||
self.paged_kv_indptr_cpu_buffer = torch.zeros_like(
|
||||
self.paged_kv_indptr.cpu, pin_memory=self.pin_memory
|
||||
) # Extra buffer for mutable paged_kv_indptr.cpu in cuda graph mode
|
||||
self.paged_kv_indices = self._make_buffer(max_num_pages)
|
||||
self.paged_kv_last_page_len = self._make_buffer(max_num_reqs)
|
||||
self.paged_kv_indptr = CpuGpuBuffer(
|
||||
max_num_reqs + 1, dtype=torch.int32, device=self.device, pin_memory=False
|
||||
)
|
||||
self.paged_kv_indices = torch.zeros(
|
||||
max_num_pages, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.paged_kv_last_page_len = CpuGpuBuffer(
|
||||
max_num_reqs, dtype=torch.int32, device=self.device, pin_memory=False
|
||||
)
|
||||
|
||||
@property
|
||||
def kv_cache_layout(self) -> KVCacheLayout:
|
||||
@@ -970,17 +969,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
return FlashInferBackend.get_dtype_for_flashinfer("fp8_e4m3")
|
||||
return self.kv_cache_spec.dtype
|
||||
|
||||
def _make_buffer(
|
||||
self, *size: int | torch.SymInt, dtype: torch.dtype = torch.int32
|
||||
) -> CpuGpuBuffer:
|
||||
return CpuGpuBuffer(
|
||||
*size,
|
||||
dtype=dtype,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
with_numpy=True,
|
||||
)
|
||||
|
||||
@override # type: ignore[misc]
|
||||
@classmethod
|
||||
def get_cudagraph_support(
|
||||
@@ -1197,7 +1185,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
if decode_wrapper is None:
|
||||
if use_cudagraph:
|
||||
paged_kv_indptr = self.paged_kv_indptr.gpu[: batch_size + 1]
|
||||
paged_kv_indices = self.paged_kv_indices.gpu
|
||||
paged_kv_indices = self.paged_kv_indices
|
||||
paged_kv_last_page_len = self.paged_kv_last_page_len.gpu[:batch_size]
|
||||
else:
|
||||
paged_kv_indptr = None
|
||||
@@ -1260,20 +1248,15 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
dtype=np.int32,
|
||||
out=self.paged_kv_indptr.np[1 : num_reqs + 1],
|
||||
)
|
||||
# NOTE(woosuk): Because self.paged_kv_indptr_cpu can be modified
|
||||
# after this line (e.g., for cuda graphs), we need to copy the data to
|
||||
# self.paged_kv_indptr_buffer to avoid race condition.
|
||||
self.paged_kv_indptr_cpu_buffer[: num_reqs + 1] = self.paged_kv_indptr.cpu[
|
||||
: num_reqs + 1
|
||||
]
|
||||
paged_kv_indptr = self.paged_kv_indptr.gpu[: num_reqs + 1]
|
||||
paged_kv_indptr.copy_(
|
||||
self.paged_kv_indptr_cpu_buffer[: num_reqs + 1], non_blocking=True
|
||||
)
|
||||
paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_reqs + 1]
|
||||
if PIN_MEMORY:
|
||||
paged_kv_indptr_cpu = paged_kv_indptr_cpu.pin_memory()
|
||||
paged_kv_indptr.copy_(paged_kv_indptr_cpu, non_blocking=True)
|
||||
|
||||
# write self.paged_kv_indices inplace
|
||||
num_actual_pages = self.paged_kv_indptr.np[num_reqs]
|
||||
paged_kv_indices = self.paged_kv_indices.gpu[:num_actual_pages]
|
||||
paged_kv_indices = self.paged_kv_indices[:num_actual_pages]
|
||||
_copy_page_indices_kernel[(num_reqs,)](
|
||||
paged_kv_indices,
|
||||
block_table_tensor,
|
||||
@@ -1289,8 +1272,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
page_size,
|
||||
paged_kv_last_page_len_np,
|
||||
)
|
||||
paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[:num_reqs]
|
||||
if PIN_MEMORY:
|
||||
paged_kv_last_page_len_cpu = paged_kv_last_page_len_cpu.pin_memory()
|
||||
self.paged_kv_last_page_len.gpu[:num_reqs].copy_(
|
||||
self.paged_kv_last_page_len.cpu[:num_reqs], non_blocking=True
|
||||
paged_kv_last_page_len_cpu, non_blocking=True
|
||||
)
|
||||
return paged_kv_indices
|
||||
|
||||
|
||||
+12
-4
@@ -11,6 +11,7 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.compilation.cuda_graph import CUDAGraphStat
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -182,11 +183,18 @@ class LogprobsTensors(NamedTuple):
|
||||
"""Create empty LogprobsTensors on CPU."""
|
||||
|
||||
logprob_token_ids = torch.empty(
|
||||
(num_positions, num_tokens_per_position), dtype=torch.int32, device="cpu"
|
||||
(num_positions, num_tokens_per_position),
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
logprobs = torch.empty_like(logprob_token_ids, dtype=torch.float32)
|
||||
selected_token_ranks = torch.empty(
|
||||
num_positions, dtype=torch.int32, device="cpu"
|
||||
logprobs = logprob_token_ids.new_empty(
|
||||
(num_positions, num_tokens_per_position),
|
||||
dtype=torch.float32,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
selected_token_ranks = logprob_token_ids.new_empty(
|
||||
num_positions, pin_memory=PIN_MEMORY
|
||||
)
|
||||
return LogprobsTensors(
|
||||
logprob_token_ids=logprob_token_ids,
|
||||
|
||||
@@ -215,7 +215,10 @@ class EncoderRunner:
|
||||
num_computed_tokens = num_computed_tokens + draft_lookahead
|
||||
|
||||
is_mm_embed = torch.zeros(
|
||||
total_num_scheduled_tokens, dtype=torch.bool, device="cpu"
|
||||
total_num_scheduled_tokens,
|
||||
dtype=torch.bool,
|
||||
device="cpu",
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
|
||||
# Whether to gather media embeddings this step.
|
||||
|
||||
Reference in New Issue
Block a user