[Misc]Use a platform independent interface to obtain the device attributes (#17100)

This commit is contained in:
ponix-j
2025-04-29 06:59:13 +00:00
committed by GitHub
parent ebb3930d28
commit bdb2cddafc
3 changed files with 8 additions and 4 deletions
+3 -2
View File
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
SamplerOutput,
SamplingMetadata, get_logprobs,
get_pythonized_sample_results)
from vllm.platforms import current_platform
from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
Logprob, SequenceGroupMetadata, SequenceOutput)
from vllm.utils import PyObjectCache, async_tensor_h2d, current_stream
@@ -158,8 +159,8 @@ class StatefulModelInput(BroadcastableModelInput):
is_first_multi_step: bool = False
base_output_proc_callback: Optional[Callable] = None
# ping-pong data structures for multi-step to wait on the previous step
step_cuda_events: List[torch.cuda.Event] = field(
default_factory=lambda: [torch.cuda.Event(blocking=True)] * 2)
step_cuda_events: List[current_platform.Event] = field(
default_factory=lambda: [current_platform.Event(blocking=True)] * 2)
num_seqs: int = -1
num_queries: int = -1
num_single_step_prefills: int = 0