[Hardware][Neuron] Refactor neuron support (#3471)
This commit is contained in:
@@ -4,11 +4,11 @@ from typing import Dict, List, Optional, Tuple
|
||||
import torch
|
||||
import random
|
||||
|
||||
from vllm.sampling_params import SamplingParams, SamplingType
|
||||
from vllm.sequence import SequenceData
|
||||
from vllm.utils import in_wsl, is_neuron
|
||||
from vllm.model_executor.layers.ops.sample import (
|
||||
get_num_triton_sampler_splits)
|
||||
from vllm.sampling_params import SamplingParams, SamplingType
|
||||
from vllm.sequence import SequenceData
|
||||
from vllm.utils import is_pin_memory_available
|
||||
|
||||
_SAMPLING_EPS = 1e-5
|
||||
_SEED_0_REPLACEMENT = 3403598558
|
||||
@@ -213,7 +213,7 @@ class SamplingTensors:
|
||||
dtype: torch.dtype) -> "SamplingTensors":
|
||||
# Note that the performance will be very bad without
|
||||
# pinned memory.
|
||||
pin_memory = not in_wsl() and not is_neuron()
|
||||
pin_memory = is_pin_memory_available()
|
||||
prompt_max_len = max(len(tokens) for tokens in prompt_tokens)
|
||||
prompt_padded_tokens = [
|
||||
tokens + [vocab_size] * (prompt_max_len - len(tokens))
|
||||
|
||||
Reference in New Issue
Block a user