[CPU] Skip Triton kernel monkey-patches when Triton-CPU is available (#44991)
Signed-off-by: jmamou <jonathan.mamou@intel.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
7299e6509e
commit
351c72d6e5
@@ -732,7 +732,11 @@ def rejection_greedy_sample_kernel(
|
||||
# Early exit for non-greedy sampling requests.
|
||||
return
|
||||
|
||||
start_idx = 0 if req_idx == 0 else tl.load(cu_num_draft_tokens_ptr + req_idx - 1)
|
||||
start_idx = (
|
||||
tl.zeros([], dtype=cu_num_draft_tokens_ptr.dtype.element_ty)
|
||||
if req_idx == 0
|
||||
else tl.load(cu_num_draft_tokens_ptr + req_idx - 1)
|
||||
)
|
||||
end_idx = tl.load(cu_num_draft_tokens_ptr + req_idx)
|
||||
num_draft_tokens = end_idx - start_idx
|
||||
|
||||
@@ -788,7 +792,11 @@ def rejection_random_sample_kernel(
|
||||
# Early exit for greedy sampling requests.
|
||||
return
|
||||
|
||||
start_idx = 0 if req_idx == 0 else tl.load(cu_num_draft_tokens_ptr + req_idx - 1)
|
||||
start_idx = (
|
||||
tl.zeros([], dtype=cu_num_draft_tokens_ptr.dtype.element_ty)
|
||||
if req_idx == 0
|
||||
else tl.load(cu_num_draft_tokens_ptr + req_idx - 1)
|
||||
)
|
||||
end_idx = tl.load(cu_num_draft_tokens_ptr + req_idx)
|
||||
num_draft_tokens = end_idx - start_idx
|
||||
|
||||
@@ -844,8 +852,8 @@ def expand_kernel(
|
||||
MAX_NUM_TOKENS: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
if req_idx == 0: # noqa: SIM108
|
||||
start_idx = 0
|
||||
if req_idx == 0:
|
||||
start_idx = tl.zeros([], dtype=cu_num_tokens_ptr.dtype.element_ty)
|
||||
else:
|
||||
start_idx = tl.load(cu_num_tokens_ptr + req_idx - 1)
|
||||
end_idx = tl.load(cu_num_tokens_ptr + req_idx)
|
||||
@@ -871,7 +879,11 @@ def sample_recovered_tokens_kernel(
|
||||
USE_FP64_GUMBEL: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
start_idx = 0 if req_idx == 0 else tl.load(cu_num_draft_tokens_ptr + req_idx - 1)
|
||||
start_idx = (
|
||||
tl.zeros([], dtype=cu_num_draft_tokens_ptr.dtype.element_ty)
|
||||
if req_idx == 0
|
||||
else tl.load(cu_num_draft_tokens_ptr + req_idx - 1)
|
||||
)
|
||||
end_idx = tl.load(cu_num_draft_tokens_ptr + req_idx)
|
||||
num_draft_tokens = end_idx - start_idx
|
||||
|
||||
|
||||
@@ -156,7 +156,6 @@ def eagle_prepare_inputs_padded_kernel(
|
||||
# cumulative sum (first entry is the first value, not zero).
|
||||
cu_draft_curr = tl.load(cu_num_draft_tokens_ptr + req_idx)
|
||||
|
||||
num_draft_tokens = 0
|
||||
if req_idx == 0:
|
||||
num_draft_tokens = cu_draft_curr
|
||||
else:
|
||||
|
||||
@@ -60,6 +60,15 @@ class CPUModelRunner(GPUModelRunner):
|
||||
v.gpu = v.cpu
|
||||
|
||||
def _postprocess_triton(self) -> None:
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
if HAS_TRITON:
|
||||
logger.info(
|
||||
"Triton-CPU backend is available; skipping C++ monkey-patches "
|
||||
"for Triton kernels."
|
||||
)
|
||||
return
|
||||
|
||||
import vllm.v1.worker.block_table
|
||||
|
||||
vllm.v1.worker.block_table._compute_slot_mapping_kernel = (
|
||||
|
||||
Reference in New Issue
Block a user