[Bugfix] Add replacement of _compute_slot_mapping_kernel on CPU (#37987)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -6,6 +6,7 @@ from typing import Any
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.utils.cpu_triton_utils as cpu_tl
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
@@ -28,6 +29,7 @@ class CPUModelRunner(GPUModelRunner):
|
||||
self.cascade_attn_enabled = False
|
||||
|
||||
self._postprocess_tensors()
|
||||
self._postprocess_triton()
|
||||
|
||||
def _postprocess_tensors(self) -> None:
|
||||
# Note: replace device tensors with cpu tensors
|
||||
@@ -52,6 +54,13 @@ class CPUModelRunner(GPUModelRunner):
|
||||
if isinstance(v, CpuGpuBuffer):
|
||||
v.gpu = v.cpu
|
||||
|
||||
def _postprocess_triton(self) -> None:
|
||||
import vllm.v1.worker.block_table
|
||||
|
||||
vllm.v1.worker.block_table._compute_slot_mapping_kernel = (
|
||||
cpu_tl.compute_slot_mapping_kernel
|
||||
)
|
||||
|
||||
@instrument(span_name="Loading (CPU)")
|
||||
def load_model(self, load_dummy_weights: bool = False) -> None:
|
||||
if load_dummy_weights:
|
||||
|
||||
Reference in New Issue
Block a user