[CPU] V1 support for the CPU backend (#16441)
This commit is contained in:
@@ -5,7 +5,7 @@ import copy
|
||||
import gc
|
||||
import time
|
||||
import weakref
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -38,7 +38,6 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
|
||||
GiB_bytes, LazyLoader, async_tensor_h2d, cdiv,
|
||||
check_use_alibi, is_pin_memory_available)
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
|
||||
from vllm.v1.core.encoder_cache_manager import compute_encoder_budget
|
||||
from vllm.v1.kv_cache_interface import (AttentionSpec, FullAttentionSpec,
|
||||
@@ -203,8 +202,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.vllm_config.compilation_config.cudagraph_capture_sizes))
|
||||
|
||||
# Cache the device properties.
|
||||
self.device_properties = torch.cuda.get_device_properties(self.device)
|
||||
self.num_sms = self.device_properties.multi_processor_count
|
||||
self._init_device_properties()
|
||||
|
||||
# Persistent buffers for CUDA graphs.
|
||||
self.input_ids = torch.zeros(self.max_num_tokens,
|
||||
@@ -315,6 +313,17 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.input_batch, scheduler_output)
|
||||
return batch_reordered
|
||||
|
||||
# Note: used for model runner override.
|
||||
def _init_device_properties(self) -> None:
|
||||
"""Initialize attributes from torch.cuda.get_device_properties
|
||||
"""
|
||||
self.device_properties = torch.cuda.get_device_properties(self.device)
|
||||
self.num_sms = self.device_properties.multi_processor_count
|
||||
|
||||
# Note: used for model runner override.
|
||||
def _sync_device(self) -> None:
|
||||
torch.cuda.synchronize()
|
||||
|
||||
def _update_states(self, scheduler_output: "SchedulerOutput") -> None:
|
||||
"""Update the cached states and the persistent batch with the scheduler
|
||||
output.
|
||||
@@ -538,8 +547,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def _prepare_inputs(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
) -> tuple[dict[str, FlashAttentionMetadata], torch.Tensor,
|
||||
Optional[SpecDecodeMetadata]]:
|
||||
) -> tuple[dict[str, Any], torch.Tensor, Optional[SpecDecodeMetadata]]:
|
||||
total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
assert total_num_scheduled_tokens > 0
|
||||
num_reqs = self.input_batch.num_reqs
|
||||
@@ -652,7 +660,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
common_attn_metadata = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc, seq_lens=seq_lens)
|
||||
|
||||
attn_metadata: dict[str, FlashAttentionMetadata] = {}
|
||||
attn_metadata: dict[str, Any] = {}
|
||||
# Prepare the attention metadata for each KV cache group and make layers
|
||||
# in the same group share the same metadata.
|
||||
for kv_cache_group_id, kv_cache_group_spec in enumerate(
|
||||
@@ -1710,7 +1718,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
# Must synchronize the non-blocking GPU->CPU transfers.
|
||||
if prompt_logprobs_dict:
|
||||
torch.cuda.synchronize()
|
||||
self._sync_device()
|
||||
|
||||
return prompt_logprobs_dict
|
||||
|
||||
@@ -1740,7 +1748,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
dtype=np.int32)
|
||||
|
||||
if skip_attn:
|
||||
attn_metadata: Optional[dict[str, FlashAttentionMetadata]] = None
|
||||
attn_metadata: Optional[dict[str, Any]] = None
|
||||
else:
|
||||
query_start_loc = self.query_start_loc[:num_reqs + 1]
|
||||
seq_lens = self.seq_lens[:num_reqs]
|
||||
@@ -1964,7 +1972,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
sampler_output = self._dummy_sampler_run(hidden_states)
|
||||
else:
|
||||
sampler_output = None
|
||||
torch.cuda.synchronize()
|
||||
self._sync_device()
|
||||
del hidden_states, sampler_output
|
||||
self.encoder_cache.clear()
|
||||
gc.collect()
|
||||
|
||||
Reference in New Issue
Block a user