[Misc]Use a platform independent interface to obtain the device attributes (#17100)
This commit is contained in:
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
|
||||
SamplerOutput,
|
||||
SamplingMetadata, get_logprobs,
|
||||
get_pythonized_sample_results)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
|
||||
Logprob, SequenceGroupMetadata, SequenceOutput)
|
||||
from vllm.utils import PyObjectCache, async_tensor_h2d, current_stream
|
||||
@@ -158,8 +159,8 @@ class StatefulModelInput(BroadcastableModelInput):
|
||||
is_first_multi_step: bool = False
|
||||
base_output_proc_callback: Optional[Callable] = None
|
||||
# ping-pong data structures for multi-step to wait on the previous step
|
||||
step_cuda_events: List[torch.cuda.Event] = field(
|
||||
default_factory=lambda: [torch.cuda.Event(blocking=True)] * 2)
|
||||
step_cuda_events: List[current_platform.Event] = field(
|
||||
default_factory=lambda: [current_platform.Event(blocking=True)] * 2)
|
||||
num_seqs: int = -1
|
||||
num_queries: int = -1
|
||||
num_single_step_prefills: int = 0
|
||||
|
||||
Reference in New Issue
Block a user