[Hardware][CPU] Support chunked-prefill and prefix-caching on CPU (#10355)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
+214
-272
@@ -2,8 +2,8 @@ import dataclasses
|
||||
import weakref
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type,
|
||||
TypeVar, Union)
|
||||
from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Type, TypeVar,
|
||||
Union)
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -19,7 +19,6 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
|
||||
MultiModalKwargs, MultiModalPlaceholderMap)
|
||||
from vllm.sequence import (IntermediateTensors, SequenceData,
|
||||
SequenceGroupMetadata)
|
||||
from vllm.utils import make_tensor_with_pad
|
||||
from vllm.worker.model_runner_base import (
|
||||
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
|
||||
_add_attn_metadata_broadcastable_dict,
|
||||
@@ -104,65 +103,223 @@ class ModelInputForCPUWithSamplingMetadata(ModelInputForCPU):
|
||||
|
||||
class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
|
||||
|
||||
class ModelInputData:
|
||||
|
||||
def __init__(self, use_mrope: bool):
|
||||
self.use_mrope = use_mrope
|
||||
self.input_tokens: List[int] = []
|
||||
self.input_positions: Optional[
|
||||
List[int]] = [] if not self.use_mrope else None
|
||||
self.seq_lens: List[int] = []
|
||||
self.query_lens: List[int] = []
|
||||
self.prefill_block_tables: List[List[int]] = []
|
||||
self.decode_block_tables: List[List[int]] = []
|
||||
self.max_decode_seq_len: int = 0
|
||||
self.num_prefills: int = 0
|
||||
self.num_prefill_tokens: int = 0
|
||||
self.num_decode_tokens: int = 0
|
||||
self.slot_mapping: List[int] = []
|
||||
self.multi_modal_inputs_list: List[MultiModalKwargs] = []
|
||||
self.multi_modal_placeholder_maps: Dict[
|
||||
str, MultiModalPlaceholderMap] = defaultdict(
|
||||
MultiModalPlaceholderMap)
|
||||
self.input_mrope_positions: Optional[List[List[int]]] = [
|
||||
[] for _ in range(3)
|
||||
] if self.use_mrope else None
|
||||
|
||||
def __init__(self,
|
||||
runner: "CPUModelRunner",
|
||||
finished_requests_ids: Optional[List[str]] = None) -> None:
|
||||
super().__init__()
|
||||
self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
|
||||
self.runner = runner
|
||||
|
||||
self.chunked_prefill = (runner.scheduler_config.chunked_prefill_enabled
|
||||
or runner.cache_config.enable_prefix_caching)
|
||||
self.model_input_cls = self.runner._model_input_cls
|
||||
self.attn_backend = self.runner.attn_backend
|
||||
self.sliding_window = self.runner.sliding_window
|
||||
self.block_size = self.runner.block_size
|
||||
self.device = self.runner.device
|
||||
self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
|
||||
self.input_data = ModelInputForCPUBuilder.ModelInputData(
|
||||
self.runner.model_config.uses_mrope)
|
||||
self.att_metadata_builder = self.runner.attn_backend.get_builder_cls()(
|
||||
self)
|
||||
|
||||
def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
|
||||
self.seq_group_metadata_list.append(seq_group_metadata)
|
||||
|
||||
def set_seq_group_list(
|
||||
self, seq_group_metadata_list: List[SequenceGroupMetadata]):
|
||||
self.seq_group_metadata_list = seq_group_metadata_list
|
||||
|
||||
def build(self) -> ModelInputForCPU:
|
||||
self._build_input_data()
|
||||
|
||||
input_data = self.input_data
|
||||
input_tokens = torch.tensor(input_data.input_tokens,
|
||||
dtype=torch.long,
|
||||
device="cpu")
|
||||
input_positions = torch.tensor(
|
||||
input_data.input_positions
|
||||
if not input_data.use_mrope else input_data.input_mrope_positions,
|
||||
dtype=torch.long,
|
||||
device="cpu")
|
||||
|
||||
# For multi-modal models
|
||||
multi_modal_kwargs = None
|
||||
# NOTE: We assume that all sequences in the group are all prompts or
|
||||
# all decodes.
|
||||
is_prompt = self.seq_group_metadata_list[0].is_prompt
|
||||
# Prepare input tensors.
|
||||
if is_prompt:
|
||||
(input_tokens, input_positions, attn_metadata, seq_lens,
|
||||
multi_modal_kwargs) = self._prepare_prompt(
|
||||
self.seq_group_metadata_list)
|
||||
else:
|
||||
(input_tokens, input_positions,
|
||||
attn_metadata) = self._prepare_decode(
|
||||
self.seq_group_metadata_list)
|
||||
seq_lens = None
|
||||
if len(input_data.multi_modal_inputs_list) != 0:
|
||||
multi_modal_kwargs = MultiModalKwargs.batch(
|
||||
input_data.multi_modal_inputs_list)
|
||||
|
||||
attn_metadata = self.att_metadata_builder.build(
|
||||
input_data.seq_lens, input_data.query_lens, -1, -1)
|
||||
|
||||
return self.model_input_cls(
|
||||
input_tokens=input_tokens,
|
||||
input_positions=input_positions,
|
||||
seq_lens=input_data.seq_lens,
|
||||
query_lens=input_data.query_lens,
|
||||
attn_metadata=attn_metadata,
|
||||
multi_modal_kwargs=multi_modal_kwargs,
|
||||
# query_lens is not needed if chunked prefill is not
|
||||
# supported. Since CPU worker doesn't support chunked prefill
|
||||
# just use seq_lens instead.
|
||||
seq_lens=seq_lens,
|
||||
query_lens=seq_lens,
|
||||
)
|
||||
|
||||
def _compute_multi_modal_input(
|
||||
self,
|
||||
seq_data: SequenceData,
|
||||
computed_len: int,
|
||||
seq_group_metadata: SequenceGroupMetadata,
|
||||
):
|
||||
def _build_input_data(self):
|
||||
for seq_group_metadata in self.seq_group_metadata_list:
|
||||
for seq_id, seq_data in seq_group_metadata.seq_data.items():
|
||||
if seq_group_metadata.is_prompt:
|
||||
self._compute_prompt_input_tokens(self.input_data,
|
||||
seq_group_metadata,
|
||||
seq_data, seq_id)
|
||||
if seq_group_metadata.multi_modal_data:
|
||||
self._compute_multi_modal_input(
|
||||
seq_group_metadata, seq_data)
|
||||
else:
|
||||
self._compute_decode_input_tokens(self.input_data,
|
||||
seq_group_metadata,
|
||||
seq_data, seq_id)
|
||||
|
||||
def _compute_decode_input_tokens(self, data: ModelInputData,
|
||||
seq_group_metadata: SequenceGroupMetadata,
|
||||
seq_data: SequenceData, seq_id: int):
|
||||
"""
|
||||
Compute decode input tokens, positions, block table and slot mapping.
|
||||
"""
|
||||
block_size = self.runner.block_size
|
||||
|
||||
block_table = seq_group_metadata.block_tables[seq_id]
|
||||
seq_len = seq_data.get_len()
|
||||
context_len = seq_data.get_num_computed_tokens()
|
||||
|
||||
tokens = seq_data.get_last_token_id()
|
||||
token_positions = seq_len - 1
|
||||
block_number = block_table[token_positions // block_size]
|
||||
block_offset = token_positions % block_size
|
||||
slot = block_number * block_size + block_offset
|
||||
|
||||
# For paged_attention kernel
|
||||
if self.runner.sliding_window:
|
||||
start_idx = max(0, seq_len - self.runner.sliding_window)
|
||||
start_block = start_idx // block_size
|
||||
start_idx = start_block * block_size
|
||||
seq_len = seq_len - start_idx
|
||||
block_table = block_table[start_block:]
|
||||
|
||||
# For MRotaryEmbedding
|
||||
if data.input_positions is None:
|
||||
next_pos = MRotaryEmbedding.get_next_input_positions(
|
||||
seq_data.mrope_position_delta,
|
||||
context_len,
|
||||
seq_len,
|
||||
)
|
||||
for idx in range(3):
|
||||
data.input_mrope_positions[idx].extend( # type: ignore
|
||||
next_pos[idx])
|
||||
else:
|
||||
data.input_positions.append(token_positions) # type: ignore
|
||||
|
||||
# Update fields
|
||||
data.input_tokens.append(tokens)
|
||||
data.max_decode_seq_len = max(data.max_decode_seq_len, seq_len)
|
||||
data.num_decode_tokens += 1
|
||||
data.slot_mapping.append(slot)
|
||||
data.decode_block_tables.append(block_table)
|
||||
data.query_lens.append(1)
|
||||
data.seq_lens.append(seq_len)
|
||||
|
||||
def _compute_prompt_input_tokens(self, data: ModelInputData,
|
||||
seq_group_metadata: SequenceGroupMetadata,
|
||||
seq_data: SequenceData, seq_id: int):
|
||||
"""
|
||||
Compute prompt input tokens, positions, block table and slot mapping.
|
||||
"""
|
||||
token_chunk_size = seq_group_metadata.token_chunk_size
|
||||
block_size = self.runner.block_size
|
||||
|
||||
block_table = seq_group_metadata.block_tables[seq_id]
|
||||
seq_len = seq_data.get_len()
|
||||
context_len = seq_data.get_num_computed_tokens()
|
||||
seq_len = min(seq_len, context_len + token_chunk_size)
|
||||
|
||||
# For prefix caching
|
||||
prefix_cache_block_num = len(seq_group_metadata.computed_block_nums)
|
||||
if prefix_cache_block_num > 0:
|
||||
prefix_cache_len = (prefix_cache_block_num *
|
||||
self.runner.block_size)
|
||||
if prefix_cache_len <= context_len:
|
||||
# We already passed the cache hit region,
|
||||
# so do normal computation.
|
||||
pass
|
||||
elif context_len < prefix_cache_len < seq_len:
|
||||
# Partial hit. Compute the missing part.
|
||||
context_len = prefix_cache_len
|
||||
token_chunk_size = seq_len - context_len
|
||||
elif seq_len <= prefix_cache_len:
|
||||
# Full hit. Only compute the last token to avoid
|
||||
# erroneous behavior. FIXME: Ideally we should directly
|
||||
# mark all tokens as computed in the scheduler and do not
|
||||
# schedule this sequence, so this case should not happen.
|
||||
context_len = seq_len - 1
|
||||
token_chunk_size = 1
|
||||
|
||||
tokens = seq_data.get_token_ids()
|
||||
tokens = tokens[context_len:seq_len]
|
||||
token_positions = range(context_len, seq_len)
|
||||
|
||||
# For encoder-only models, the block_table is None,
|
||||
# and there is no need to initialize the slot_mapping.
|
||||
if block_table is not None:
|
||||
slot_mapping = [_PAD_SLOT_ID] * len(token_positions)
|
||||
for i, pos in enumerate(token_positions):
|
||||
block_number = block_table[pos // block_size]
|
||||
block_offset = pos % block_size
|
||||
slot = block_number * block_size + block_offset
|
||||
slot_mapping[i] = slot
|
||||
data.slot_mapping.extend(slot_mapping)
|
||||
|
||||
# The MROPE positions are prepared in _compute_multi_modal_input
|
||||
if data.input_positions is not None:
|
||||
data.input_positions.extend(token_positions)
|
||||
|
||||
# Update fields
|
||||
data.input_tokens.extend(tokens)
|
||||
data.num_prefills += 1
|
||||
data.num_prefill_tokens += len(tokens)
|
||||
data.query_lens.append(len(tokens))
|
||||
data.prefill_block_tables.append(block_table)
|
||||
data.seq_lens.append(seq_len)
|
||||
|
||||
def _compute_multi_modal_input(self,
|
||||
seq_group_metadata: SequenceGroupMetadata,
|
||||
seq_data: SequenceData):
|
||||
computed_len = seq_data.get_num_computed_tokens()
|
||||
seq_len = self.input_data.seq_lens[-1]
|
||||
|
||||
# NOTE: mm_data only includes the subset of multi-modal items that
|
||||
# intersect with the current prefill positions.
|
||||
mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
|
||||
seq_group_metadata,
|
||||
range(computed_len, len(seq_data.get_token_ids())),
|
||||
)
|
||||
seq_group_metadata, range(computed_len, seq_len))
|
||||
|
||||
if not mm_data:
|
||||
return None, None, None
|
||||
return
|
||||
|
||||
if self.runner.mm_registry.has_processor(self.runner.model_config):
|
||||
mm_kwargs = mm_data
|
||||
@@ -173,8 +330,10 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
|
||||
)
|
||||
|
||||
# special processing for mrope position deltas.
|
||||
mrope_positions = None
|
||||
if self.runner.model_config.uses_mrope:
|
||||
assert not self.chunked_prefill, \
|
||||
"MROPE on CPU does not support chunked-prefill."
|
||||
|
||||
image_grid_thw = mm_kwargs.get("image_grid_thw", None)
|
||||
video_grid_thw = mm_kwargs.get("video_grid_thw", None)
|
||||
assert image_grid_thw is not None or video_grid_thw is not None, (
|
||||
@@ -198,226 +357,15 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
|
||||
context_len=computed_len,
|
||||
)
|
||||
seq_data.mrope_position_delta = mrope_position_delta
|
||||
return mm_kwargs, placeholder_maps, mrope_positions
|
||||
|
||||
def _prepare_prompt(
|
||||
self,
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
|
||||
BatchedTensorInputs]:
|
||||
assert len(seq_group_metadata_list) > 0
|
||||
input_tokens: List[int] = []
|
||||
input_positions: List[int] = []
|
||||
input_mrope_positions: List[List[int]] = [[] for _ in range(3)]
|
||||
for i in range(3):
|
||||
self.input_data.input_mrope_positions[ # type: ignore
|
||||
i].extend(mrope_positions[i])
|
||||
|
||||
slot_mapping: List[int] = []
|
||||
seq_lens: List[int] = []
|
||||
multi_modal_kwargs_list: List[MultiModalKwargs] = []
|
||||
multi_modal_placeholder_maps: Dict[
|
||||
str,
|
||||
MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
|
||||
|
||||
for seq_group_metadata in seq_group_metadata_list:
|
||||
assert seq_group_metadata.is_prompt
|
||||
seq_ids = list(seq_group_metadata.seq_data.keys())
|
||||
assert len(seq_ids) == 1
|
||||
seq_id = seq_ids[0]
|
||||
|
||||
seq_data = seq_group_metadata.seq_data[seq_id]
|
||||
prompt_tokens = seq_data.get_token_ids()
|
||||
computed_len = seq_data.get_num_computed_tokens()
|
||||
seq_len = len(prompt_tokens)
|
||||
|
||||
seq_lens.append(seq_len) # Prompt token num
|
||||
input_tokens.extend(prompt_tokens) # Token ids
|
||||
|
||||
mrope_positions = None
|
||||
if seq_group_metadata.multi_modal_data:
|
||||
(
|
||||
mm_kwargs,
|
||||
placeholder_maps,
|
||||
mrope_positions,
|
||||
) = self._compute_multi_modal_input(seq_data, computed_len,
|
||||
seq_group_metadata)
|
||||
|
||||
multi_modal_kwargs_list.append(mm_kwargs)
|
||||
for modality, placeholder_map in placeholder_maps.items():
|
||||
multi_modal_placeholder_maps[modality].extend(
|
||||
placeholder_map)
|
||||
|
||||
# Token position ids
|
||||
# NOTE(woosuk): Here we assume that the first token in the prompt
|
||||
# is always the first token in the sequence.
|
||||
if mrope_positions:
|
||||
for idx in range(3):
|
||||
input_mrope_positions[idx].extend(mrope_positions[idx])
|
||||
else:
|
||||
input_positions.extend(list(range(computed_len, seq_len)))
|
||||
|
||||
# Compute the slot mapping.
|
||||
block_table = seq_group_metadata.block_tables[seq_id]
|
||||
# Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
|
||||
# where start_idx is max(0, seq_len - sliding_window).
|
||||
# For example, if the prompt len is 10, sliding window is 8, and
|
||||
# block size is 4, the first two tokens are masked and the slot
|
||||
# mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
|
||||
start_idx = 0
|
||||
if self.sliding_window is not None:
|
||||
start_idx = max(0, seq_len - self.sliding_window)
|
||||
|
||||
for i in range(computed_len, seq_len):
|
||||
if i < start_idx:
|
||||
slot_mapping.append(_PAD_SLOT_ID)
|
||||
continue
|
||||
|
||||
# For encoder-only models, the block_table is None,
|
||||
# and there is no need to initialize the slot_mapping.
|
||||
if block_table is not None:
|
||||
block_number = block_table[i //
|
||||
self.block_size] # type: ignore
|
||||
block_offset = i % self.block_size # type: ignore
|
||||
slot = block_number * self.block_size + block_offset
|
||||
slot_mapping.append(slot)
|
||||
|
||||
if any(input_mrope_positions):
|
||||
input_positions = None # type: ignore
|
||||
else:
|
||||
input_mrope_positions = None # type: ignore
|
||||
|
||||
num_prompt_tokens = len(input_tokens)
|
||||
|
||||
input_tokens = torch.tensor(input_tokens,
|
||||
dtype=torch.long,
|
||||
device=self.device) # type: ignore
|
||||
input_positions = torch.tensor(input_positions
|
||||
or input_mrope_positions,
|
||||
dtype=torch.long,
|
||||
device=self.device) # type: ignore
|
||||
slot_mapping = torch.tensor(slot_mapping,
|
||||
dtype=torch.long,
|
||||
device=self.device) # type: ignore
|
||||
placeholder_index_maps = {
|
||||
modality: placeholder_map.index_map()
|
||||
for modality, placeholder_map in
|
||||
multi_modal_placeholder_maps.items()
|
||||
}
|
||||
|
||||
attn_metadata = self.attn_backend.make_metadata(
|
||||
is_prompt=True,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_tensor=torch.tensor([]),
|
||||
max_decode_seq_len=0,
|
||||
num_prefills=len(seq_lens),
|
||||
num_prefill_tokens=num_prompt_tokens,
|
||||
num_decode_tokens=0,
|
||||
block_tables=torch.tensor([]),
|
||||
slot_mapping=slot_mapping,
|
||||
multi_modal_placeholder_index_maps=placeholder_index_maps,
|
||||
)
|
||||
|
||||
multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
|
||||
|
||||
return (input_tokens, input_positions, attn_metadata, seq_lens,
|
||||
multi_modal_kwargs)
|
||||
|
||||
def _prepare_decode(
|
||||
self,
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
|
||||
assert len(seq_group_metadata_list) > 0
|
||||
input_tokens: List[int] = []
|
||||
input_positions: List[int] = []
|
||||
input_mrope_positions: List[List[int]] = [[] for _ in range(3)]
|
||||
slot_mapping: List[int] = []
|
||||
seq_lens: List[int] = []
|
||||
block_tables: List[List[int]] = []
|
||||
|
||||
for seq_group_metadata in seq_group_metadata_list:
|
||||
assert not seq_group_metadata.is_prompt
|
||||
assert seq_group_metadata.token_chunk_size == 1
|
||||
|
||||
seq_ids = list(seq_group_metadata.seq_data.keys())
|
||||
|
||||
for seq_id in seq_ids:
|
||||
seq_data = seq_group_metadata.seq_data[seq_id]
|
||||
generation_token = seq_data.get_last_token_id()
|
||||
input_tokens.append(generation_token)
|
||||
|
||||
seq_len = seq_data.get_len()
|
||||
position = seq_len - 1
|
||||
if seq_data.mrope_position_delta is not None:
|
||||
context_len = seq_data.get_num_computed_tokens()
|
||||
next_pos = MRotaryEmbedding.get_next_input_positions(
|
||||
seq_data.mrope_position_delta,
|
||||
context_len,
|
||||
seq_len,
|
||||
)
|
||||
for idx in range(3):
|
||||
input_mrope_positions[idx].extend(next_pos[idx])
|
||||
else:
|
||||
input_positions.append(position)
|
||||
|
||||
seq_len = seq_len if self.sliding_window is None else min(
|
||||
seq_len, self.sliding_window)
|
||||
seq_lens.append(seq_len)
|
||||
|
||||
block_table = seq_group_metadata.block_tables[seq_id]
|
||||
block_number = block_table[position // self.block_size]
|
||||
block_offset = position % self.block_size
|
||||
slot = block_number * self.block_size + block_offset
|
||||
slot_mapping.append(slot)
|
||||
|
||||
if self.sliding_window is not None:
|
||||
sliding_window_blocks = (self.sliding_window //
|
||||
self.block_size)
|
||||
block_table = block_table[-sliding_window_blocks:]
|
||||
block_tables.append(block_table)
|
||||
|
||||
if any(input_mrope_positions):
|
||||
input_positions = None # type: ignore
|
||||
else:
|
||||
input_mrope_positions = None # type: ignore
|
||||
|
||||
max_decode_seq_len = max(seq_lens)
|
||||
|
||||
input_tokens = torch.tensor(input_tokens,
|
||||
dtype=torch.long,
|
||||
device=self.device)
|
||||
input_positions = torch.tensor(input_positions
|
||||
or input_mrope_positions,
|
||||
dtype=torch.long,
|
||||
device=self.device)
|
||||
slot_mapping = torch.tensor(slot_mapping,
|
||||
dtype=torch.long,
|
||||
device=self.device)
|
||||
seq_lens_tensor = torch.tensor(seq_lens,
|
||||
dtype=torch.int,
|
||||
device=self.device)
|
||||
|
||||
block_tables = make_tensor_with_pad(
|
||||
block_tables,
|
||||
pad=0,
|
||||
dtype=torch.int,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
attn_metadata = self.attn_backend.make_metadata(
|
||||
is_prompt=False,
|
||||
slot_mapping=slot_mapping,
|
||||
multi_modal_placeholder_index_maps=None,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_tensor=seq_lens_tensor,
|
||||
max_decode_seq_len=max_decode_seq_len,
|
||||
num_prefill_tokens=0,
|
||||
num_decode_tokens=len(input_tokens),
|
||||
num_prefills=0,
|
||||
block_tables=block_tables,
|
||||
)
|
||||
return (
|
||||
input_tokens,
|
||||
input_positions,
|
||||
attn_metadata,
|
||||
)
|
||||
self.input_data.multi_modal_inputs_list.append(mm_kwargs)
|
||||
for modality, placeholder_map in placeholder_maps.items():
|
||||
self.input_data.multi_modal_placeholder_maps[modality].extend(
|
||||
placeholder_map)
|
||||
|
||||
|
||||
class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
|
||||
@@ -436,8 +384,6 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
|
||||
**kwargs,
|
||||
):
|
||||
ModelRunnerBase.__init__(self, vllm_config)
|
||||
# Currently, CPU worker doesn't support chunked prefill.
|
||||
assert self.scheduler_config.chunked_prefill_enabled is False
|
||||
model_config = self.model_config
|
||||
cache_config = self.cache_config
|
||||
|
||||
@@ -479,8 +425,7 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
|
||||
|
||||
"""
|
||||
builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
|
||||
for seq_group_metadata in seq_group_metadata_list:
|
||||
builder.add_seq_group(seq_group_metadata)
|
||||
builder.set_seq_group_list(seq_group_metadata_list)
|
||||
|
||||
return builder.build() # type: ignore
|
||||
|
||||
@@ -537,22 +482,19 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
|
||||
"CPU worker does not support multi-step execution.")
|
||||
|
||||
model_executable = self.model
|
||||
execute_model_kwargs = {
|
||||
"input_ids":
|
||||
model_input.input_tokens,
|
||||
"positions":
|
||||
model_input.input_positions,
|
||||
"kv_caches":
|
||||
kv_caches,
|
||||
"attn_metadata":
|
||||
model_input.attn_metadata,
|
||||
**MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
|
||||
device=self.device),
|
||||
"intermediate_tensors":
|
||||
intermediate_tensors,
|
||||
}
|
||||
multimodal_kwargs = {}
|
||||
if model_input.multi_modal_kwargs is not None:
|
||||
multimodal_kwargs = MultiModalKwargs.as_kwargs(
|
||||
model_input.multi_modal_kwargs, device=self.device)
|
||||
|
||||
hidden_states = model_executable(**execute_model_kwargs)
|
||||
hidden_states = model_executable(
|
||||
input_ids=model_input.input_tokens,
|
||||
positions=model_input.input_positions,
|
||||
kv_caches=kv_caches,
|
||||
attn_metadata=model_input.attn_metadata,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
**multimodal_kwargs,
|
||||
)
|
||||
|
||||
# Compute the logits.
|
||||
logits = self.model.compute_logits(hidden_states,
|
||||
|
||||
Reference in New Issue
Block a user