Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
7a90eb98ab
commit
07f9baf756
@@ -132,8 +132,10 @@ def benchmark_function(
|
||||
reset_memory_stats()
|
||||
|
||||
# Benchmark
|
||||
start_events = [torch.Event(enable_timing=True) for _ in range(benchmark_iters)]
|
||||
end_events = [torch.Event(enable_timing=True) for _ in range(benchmark_iters)]
|
||||
start_events = [
|
||||
torch.cuda.Event(enable_timing=True) for _ in range(benchmark_iters)
|
||||
]
|
||||
end_events = [torch.cuda.Event(enable_timing=True) for _ in range(benchmark_iters)]
|
||||
|
||||
for i in range(benchmark_iters):
|
||||
logits_copy = logits.clone()
|
||||
|
||||
@@ -134,8 +134,8 @@ def benchmark_config(
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Benchmark
|
||||
start = torch.Event(enable_timing=True)
|
||||
end = torch.Event(enable_timing=True)
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(num_iters):
|
||||
with override_config(config):
|
||||
|
||||
@@ -170,8 +170,8 @@ def benchmark_config(
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
start = torch.Event(enable_timing=True)
|
||||
end = torch.Event(enable_timing=True)
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
latencies: list[float] = []
|
||||
for _ in range(num_iters):
|
||||
start.record()
|
||||
|
||||
@@ -33,7 +33,7 @@ def hf3fs_stats():
|
||||
def _make_cuda_event():
|
||||
"""Return a real CUDA event when available, otherwise a MagicMock."""
|
||||
if torch.cuda.is_available():
|
||||
return torch.Event()
|
||||
return torch.cuda.Event()
|
||||
return MagicMock()
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import regex as re
|
||||
# --------------------------------------------------------------------------- #
|
||||
_TORCH_CUDA_PATTERNS = [
|
||||
r"\btorch\.cuda\.(empty_cache|synchronize|device_count|current_device|memory_reserved|memory_allocated|max_memory_allocated|max_memory_reserved|reset_peak_memory_stats|memory_stats|mem_get_info|set_device|device\()\b",
|
||||
r"\btorch\.cuda\.(manual_seed|manual_seed_all|Event)\b",
|
||||
r"\btorch\.cuda\.(manual_seed|manual_seed_all)\b",
|
||||
r"\bwith\storch\.cuda\.device\b",
|
||||
# Calls torch.cuda.{_is_compiled/_device_count_amdsmi/_device_count_nvml} internally
|
||||
r"\bcuda_device_count_stateless\(\)\b",
|
||||
@@ -21,7 +21,6 @@ ALLOWED_FILES = {
|
||||
"vllm/device_allocator/",
|
||||
"vllm/distributed/weight_transfer/ipc_engine.py",
|
||||
"tests/distributed/test_packed_tensor.py",
|
||||
"tools/pre_commit/check_torch_cuda.py",
|
||||
}
|
||||
|
||||
|
||||
@@ -40,13 +39,6 @@ def scan_file(path: str) -> int:
|
||||
f"Found {matched_text} API call. Use set_random_seed instead."
|
||||
)
|
||||
return 1
|
||||
if matched_text == "torch.cuda.Event":
|
||||
print(
|
||||
f"{path}:{line_num}: "
|
||||
"\033[91merror:\033[0m "
|
||||
"Found torch.cuda.Event API call. Use torch.Event instead."
|
||||
)
|
||||
return 1
|
||||
print(
|
||||
f"{path}:{line_num}: "
|
||||
"\033[91merror:\033[0m " # red color
|
||||
|
||||
@@ -742,8 +742,8 @@ class EplbState:
|
||||
is_main_rank = ep_rank == 0
|
||||
if is_main_rank:
|
||||
if not self.is_async or is_profile:
|
||||
start_event = torch.Event(enable_timing=True)
|
||||
end_event = torch.Event(enable_timing=True)
|
||||
start_event = torch.cuda.Event(enable_timing=True)
|
||||
end_event = torch.cuda.Event(enable_timing=True)
|
||||
start_event.record()
|
||||
logger.info(
|
||||
"Rearranging experts %s %s...",
|
||||
|
||||
@@ -31,7 +31,7 @@ class CpuGpuEvent:
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._event = torch.Event()
|
||||
self._event = torch.cuda.Event()
|
||||
self._recorded = threading.Event()
|
||||
|
||||
def wait(self, stream: torch.cuda.Stream | None = None):
|
||||
@@ -56,7 +56,7 @@ class CpuGpuEvent:
|
||||
"CpuGpuEvent.record() called before the previous event was "
|
||||
"consumed by wait()"
|
||||
)
|
||||
self._event = torch.Event()
|
||||
self._event = torch.cuda.Event()
|
||||
self._event.record(stream)
|
||||
self._recorded.set()
|
||||
|
||||
|
||||
@@ -239,7 +239,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# this event is complete the request is considered "done sending"
|
||||
# by get_finished; clients block on the per-file flock to wait for
|
||||
# the disk write itself.
|
||||
self._req_copy_events: dict[str, torch.Event] = {}
|
||||
self._req_copy_events: dict[str, torch.cuda.Event] = {}
|
||||
# req_ids reported as finished-generating by the scheduler,
|
||||
# accumulated across get_finished calls.
|
||||
self._accumulated_finished_req_ids: set[str] = set()
|
||||
@@ -320,7 +320,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
@staticmethod
|
||||
def _write_tensors(
|
||||
tensors: dict[str, torch.Tensor],
|
||||
event: torch.Event,
|
||||
event: torch.cuda.Event,
|
||||
filename: str,
|
||||
lock_fd: int | None,
|
||||
) -> None:
|
||||
@@ -375,7 +375,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
copy_stream = self._get_copy_stream()
|
||||
|
||||
# Ensure the copy stream sees all prior writes on the default stream.
|
||||
ready_event = torch.Event()
|
||||
ready_event = torch.cuda.Event()
|
||||
ready_event.record()
|
||||
copy_stream.wait_event(ready_event)
|
||||
|
||||
@@ -396,7 +396,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
pinned_hs.copy_(hidden_states_gpu, non_blocking=True)
|
||||
|
||||
# Record completion of this copy on the copy stream.
|
||||
copy_done = torch.Event()
|
||||
copy_done = torch.cuda.Event()
|
||||
copy_done.record(copy_stream)
|
||||
|
||||
# token_ids is already on CPU (created in request_finished).
|
||||
|
||||
@@ -221,7 +221,7 @@ class Hf3fsClient:
|
||||
|
||||
@wsynchronized()
|
||||
def batch_write(
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.Event
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
|
||||
) -> list[int]:
|
||||
"""Write data from tensors to the file at specified offsets.
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@ class AsyncOperationManager:
|
||||
# CUDA streams for async operations
|
||||
self._save_stream = torch.cuda.Stream()
|
||||
self._load_stream = torch.cuda.Stream()
|
||||
self._save_event = torch.Event()
|
||||
self._save_event = torch.cuda.Event()
|
||||
|
||||
# Buffer allocators for data copying
|
||||
self._save_buffer_allocator = CopyBufferAllocator(
|
||||
@@ -171,7 +171,7 @@ class AsyncOperationManager:
|
||||
def submit_save_operation(self, request_id: str, block_ids, block_hashes) -> Future:
|
||||
"""Submit a save operation for async execution."""
|
||||
future: Future[Any] = Future()
|
||||
main_stream_event = torch.Event()
|
||||
main_stream_event = torch.cuda.Event()
|
||||
main_stream_event.record()
|
||||
task = (request_id, block_ids, block_hashes, future, main_stream_event)
|
||||
self._save_queue.put(task)
|
||||
@@ -304,7 +304,7 @@ class AsyncOperationManager:
|
||||
block_ids, buffers, "gather"
|
||||
)
|
||||
|
||||
save_stream_event = torch.Event()
|
||||
save_stream_event = torch.cuda.Event()
|
||||
save_stream_event.record(self._save_stream) # Record gather completion
|
||||
|
||||
# Step3: Write data in batches
|
||||
|
||||
@@ -75,7 +75,7 @@ class Hf3fsClient:
|
||||
return torch.frombuffer(buffer_data, dtype=dtype)
|
||||
|
||||
def batch_write(
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.Event
|
||||
self, offsets: list[int], tensors: list[torch.Tensor], event: torch.cuda.Event
|
||||
) -> list[int]:
|
||||
"""Write data from tensors to file at specified offsets."""
|
||||
results = []
|
||||
|
||||
+4
-4
@@ -430,7 +430,7 @@ class LMCacheMPWorkerAdapter:
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def submit_store_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.Event
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
):
|
||||
"""
|
||||
Submit a KV cache store request to LMCache
|
||||
@@ -464,7 +464,7 @@ class LMCacheMPWorkerAdapter:
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def submit_retrieve_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.Event
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
):
|
||||
"""
|
||||
Submit a KV cache retrieve request to LMCache
|
||||
@@ -501,7 +501,7 @@ class LMCacheMPWorkerAdapter:
|
||||
self,
|
||||
request_ids: list[str],
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.Event,
|
||||
event: torch.cuda.Event,
|
||||
):
|
||||
"""
|
||||
Submit a batched store request to LMCache
|
||||
@@ -550,7 +550,7 @@ class LMCacheMPWorkerAdapter:
|
||||
self,
|
||||
request_ids: list[str],
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.Event,
|
||||
event: torch.cuda.Event,
|
||||
):
|
||||
"""
|
||||
Submit a batched retrieve request to LMCache
|
||||
|
||||
@@ -589,7 +589,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return
|
||||
|
||||
with torch.cuda.stream(torch.cuda.current_stream()):
|
||||
event = torch.Event(interprocess=True)
|
||||
event = torch.cuda.Event(interprocess=True)
|
||||
event.record()
|
||||
|
||||
self.worker_adapter.batched_submit_retrieve_requests(
|
||||
@@ -663,7 +663,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return
|
||||
|
||||
with torch.cuda.stream(torch.cuda.current_stream()):
|
||||
event = torch.Event(interprocess=True)
|
||||
event = torch.cuda.Event(interprocess=True)
|
||||
event.record()
|
||||
|
||||
self.worker_adapter.batched_submit_store_requests(
|
||||
|
||||
@@ -323,7 +323,7 @@ class ReqMeta:
|
||||
can_save: bool | None = None
|
||||
load_spec: LoadSpec | None = None
|
||||
is_last_chunk: bool | None = None
|
||||
current_event: torch.Event | None = None
|
||||
current_event: torch.cuda.Event | None = None
|
||||
|
||||
token_ids: list[int] | None = None
|
||||
num_prompt_tokens: int | None = None
|
||||
|
||||
@@ -1357,7 +1357,7 @@ class MooncakeStoreWorker:
|
||||
current_event = None
|
||||
for request in meta.requests:
|
||||
if request.can_save:
|
||||
current_event = torch.Event()
|
||||
current_event = torch.cuda.Event()
|
||||
current_event.record()
|
||||
break
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ class WriteTask:
|
||||
local_block_ids: list[int]
|
||||
remote_block_ids_hint: list[int] | None
|
||||
layer_name: str
|
||||
event: torch.Event
|
||||
event: torch.cuda.Event
|
||||
remote_notify_port: int
|
||||
remote_ip: str
|
||||
enqueue_time: float = field(default_factory=time.perf_counter)
|
||||
|
||||
@@ -1061,7 +1061,7 @@ class MoRIIOConnectorWorker:
|
||||
# when mori-io supports ibgda functionality
|
||||
|
||||
stream = torch.cuda.current_stream()
|
||||
event = torch.Event()
|
||||
event = torch.cuda.Event()
|
||||
event.record(stream)
|
||||
|
||||
task = WriteTask(
|
||||
|
||||
@@ -89,7 +89,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
|
||||
vllm_config = get_current_vllm_config()
|
||||
self._lora_stream = _get_lora_aux_cuda_stream()
|
||||
assert current_platform.is_cuda_alike()
|
||||
self._events = [torch.Event(), torch.Event()]
|
||||
self._events = [torch.cuda.Event(), torch.cuda.Event()]
|
||||
# lora_linear avoids prefix conflicts with the base layer
|
||||
self.layer_name = self.base_layer.prefix + ".lora_linear_async"
|
||||
compilation_config = vllm_config.compilation_config
|
||||
|
||||
@@ -118,7 +118,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
|
||||
def _init_lora_stream_context(self) -> None:
|
||||
self._lora_stream: torch.cuda.Stream | None = None
|
||||
self._events: tuple[torch.Event, ...] | None = None
|
||||
self._events: tuple[torch.cuda.Event, ...] | None = None
|
||||
if not self._enable_aux_cuda_stream:
|
||||
return
|
||||
if not current_platform.is_cuda_alike():
|
||||
@@ -127,7 +127,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
# 4 events: 2 per (base GEMM, LoRA) pair so w13 and w2 don't reuse
|
||||
# the same event objects; reuse-within-a-pair is fine because the
|
||||
# second pair starts only after intermediate_cache1.add_() has joined.
|
||||
self._events = tuple(torch.Event() for _ in range(4))
|
||||
self._events = tuple(torch.cuda.Event() for _ in range(4))
|
||||
|
||||
def _build_lora_context(self):
|
||||
use_dual_stream = (
|
||||
|
||||
@@ -51,7 +51,7 @@ class MoELoRAContext:
|
||||
# Events are paired one-per-overlap-pair: events[0,1] for w13,
|
||||
# events[2,3] for w2, so the two pairs do not race on the same event.
|
||||
aux_stream: torch.cuda.Stream | None = None
|
||||
events: tuple[torch.Event, ...] | None = None
|
||||
events: tuple[torch.cuda.Event, ...] | None = None
|
||||
|
||||
# Per-rank token→LoRA mapping after EP dispatch. Set by
|
||||
# FusedMoEPrepareAndFinalizeModular.prepare() when EP+LoRA is active, read
|
||||
|
||||
@@ -388,7 +388,7 @@ class _ModuleOffloader:
|
||||
|
||||
# Event to signal when H2D copy to static buffer is complete.
|
||||
# Used for per-layer synchronization (both eager and capture modes).
|
||||
self._copy_done_event = torch.Event()
|
||||
self._copy_done_event = torch.cuda.Event()
|
||||
|
||||
# Track whether _copy_done_event is valid for eager-mode wait_event.
|
||||
# False when: (1) never recorded, or (2) last recorded during a
|
||||
@@ -518,7 +518,7 @@ class _ModuleOffloader:
|
||||
|
||||
# Fork: record event on compute stream, copy_stream waits on it
|
||||
# This joins copy_stream to any active CUDA graph capture
|
||||
fork_event = torch.Event()
|
||||
fork_event = torch.cuda.Event()
|
||||
torch.cuda.current_stream().record_event(fork_event)
|
||||
self.copy_stream.wait_event(fork_event)
|
||||
|
||||
|
||||
@@ -272,7 +272,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# [0]: GEMM start / post-GEMM event0. [1..3]: GEMM done events;
|
||||
# [1] doubles as post-GEMM event1. Reuse is safe: GEMM fully joins
|
||||
# before post-GEMM starts.
|
||||
self.ln_events = [torch.Event() for _ in range(4)]
|
||||
self.ln_events = [torch.cuda.Event() for _ in range(4)]
|
||||
|
||||
assert cache_config is not None, "DeepseekV4 attention requires cache_config"
|
||||
# ---- Attention / KV-cache setup ----
|
||||
@@ -760,7 +760,10 @@ class DeepseekV4Indexer(nn.Module):
|
||||
|
||||
# None on ROCm — maybe_execute_in_parallel falls back to sequential.
|
||||
self.aux_stream = aux_stream
|
||||
self.ln_events: list[torch.Event] = [torch.Event(), torch.Event()]
|
||||
self.ln_events: list[torch.cuda.Event] = [
|
||||
torch.cuda.Event(),
|
||||
torch.cuda.Event(),
|
||||
]
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -44,6 +44,17 @@ class DeepseekV4XPUAttention(DeepseekV4Attention):
|
||||
backend_cls = DeepseekV4XPUSparseBackend
|
||||
use_flashmla_fp8_layout = True
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
# torch.cuda.Event() raises RuntimeError on XPU ("dummy base class").
|
||||
# The Base and DeepseekV4Indexer both create cuda Events in __init__, so
|
||||
# we temporarily redirect torch.cuda.Event → torch.xpu.Event.
|
||||
_orig_event = torch.cuda.Event
|
||||
torch.cuda.Event = torch.xpu.Event # type: ignore[misc]
|
||||
try:
|
||||
super().__init__(*args, **kwargs)
|
||||
finally:
|
||||
torch.cuda.Event = _orig_event # type: ignore[misc]
|
||||
|
||||
def _fused_qnorm_rope_kv_insert(self, q, kv, positions, attn_metadata):
|
||||
from typing import cast
|
||||
|
||||
|
||||
@@ -20,8 +20,8 @@ class EventType(Enum):
|
||||
def maybe_execute_in_parallel(
|
||||
fn0: Callable[[], Any],
|
||||
fn1: Callable[[], Any],
|
||||
event0: torch.Event,
|
||||
event1: torch.Event,
|
||||
event0: torch.cuda.Event,
|
||||
event1: torch.cuda.Event,
|
||||
aux_stream: torch.cuda.Stream | None = None,
|
||||
) -> tuple[Any, Any]:
|
||||
"""Run two functions potentially in parallel on separate CUDA streams.
|
||||
@@ -61,8 +61,8 @@ def maybe_execute_in_parallel(
|
||||
def execute_in_parallel(
|
||||
default_fn: Callable[[], Any],
|
||||
aux_fns: list[Callable[[], Any] | None],
|
||||
start_event: torch.Event,
|
||||
done_events: list[torch.Event],
|
||||
start_event: torch.cuda.Event,
|
||||
done_events: list[torch.cuda.Event],
|
||||
aux_streams: list[torch.cuda.Stream] | None = None,
|
||||
enable: bool = False,
|
||||
) -> tuple[Any, list[Any]]:
|
||||
@@ -108,7 +108,7 @@ def execute_in_parallel(
|
||||
)
|
||||
|
||||
aux_results = [None] * len(aux_fns)
|
||||
pending: list[torch.Event] = []
|
||||
pending: list[torch.cuda.Event] = []
|
||||
|
||||
start_event.record()
|
||||
for i, fn in enumerate(aux_fns):
|
||||
|
||||
@@ -468,7 +468,7 @@ class NgramProposerGPU:
|
||||
|
||||
|
||||
def update_scheduler_for_invalid_drafts(
|
||||
num_valid_draft_tokens_event: torch.Event,
|
||||
num_valid_draft_tokens_event: torch.cuda.Event,
|
||||
num_valid_draft_tokens_cpu: torch.Tensor,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
req_id_to_index: dict[str, int],
|
||||
@@ -643,7 +643,7 @@ def _sync_num_tokens(
|
||||
def copy_num_valid_draft_tokens(
|
||||
num_valid_draft_tokens_cpu: torch.Tensor,
|
||||
num_valid_draft_tokens_copy_stream: torch.cuda.Stream,
|
||||
num_valid_draft_tokens_event: torch.Event,
|
||||
num_valid_draft_tokens_event: torch.cuda.Event,
|
||||
num_valid_draft_tokens: torch.Tensor | None,
|
||||
batch_size: int,
|
||||
) -> None:
|
||||
|
||||
@@ -48,6 +48,7 @@ def get_memory_info(*args: Any, **kwargs: Any) -> tuple[int, int]:
|
||||
|
||||
|
||||
torch.Event = _EventPlaceholder
|
||||
torch.cuda.Event = _EventPlaceholder
|
||||
torch.cuda.Stream = _StreamPlaceholder
|
||||
torch.cuda.set_stream = noop
|
||||
torch.cuda.current_stream = lambda *args, **kwargs: _StreamPlaceholder()
|
||||
|
||||
@@ -24,7 +24,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
self.model_runner_output = model_runner_output
|
||||
self.sampler_output = sampler_output
|
||||
self.num_sampled_tokens = num_sampled_tokens
|
||||
self.copy_event = torch.Event()
|
||||
self.copy_event = torch.cuda.Event()
|
||||
|
||||
with stream(copy_stream, main_stream):
|
||||
copy_stream.wait_stream(main_stream)
|
||||
@@ -81,7 +81,7 @@ class AsyncPoolingOutput(AsyncModelRunnerOutput):
|
||||
self.model_runner_output = model_runner_output
|
||||
self.pooler_output = pooler_output
|
||||
self.is_valid = is_valid
|
||||
self.copy_event = torch.Event()
|
||||
self.copy_event = torch.cuda.Event()
|
||||
|
||||
with stream(copy_stream, main_stream):
|
||||
copy_stream.wait_stream(main_stream)
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
class PendingRecv:
|
||||
"""Per-step slot data for a deferred postprocess on the main stream."""
|
||||
|
||||
event: torch.Event
|
||||
event: torch.cuda.Event
|
||||
|
||||
sampled_tokens: torch.Tensor # [num_reqs, max_sample_len]
|
||||
num_sampled: torch.Tensor # [num_reqs]
|
||||
|
||||
@@ -12,7 +12,7 @@ class DraftTokensHandler:
|
||||
def __init__(self, device: torch.device | None = None):
|
||||
self.device = device
|
||||
self.copy_stream = torch.cuda.Stream(device)
|
||||
self.copy_event = torch.Event()
|
||||
self.copy_event = torch.cuda.Event()
|
||||
|
||||
self.req_ids: list[str] = []
|
||||
self.draft_tokens_np: np.ndarray | None = None
|
||||
|
||||
@@ -860,7 +860,7 @@ class GPUModelRunner(
|
||||
# N-gram GPU path: async D2H buffer/event for per-request valid draft counts.
|
||||
self._num_valid_draft_tokens: torch.Tensor | None = None
|
||||
self._num_valid_draft_tokens_cpu: torch.Tensor | None = None
|
||||
self._num_valid_draft_tokens_event: torch.Event | None = None
|
||||
self._num_valid_draft_tokens_event: torch.cuda.Event | None = None
|
||||
self._num_valid_draft_tokens_copy_stream: torch.cuda.Stream | None = None
|
||||
if (
|
||||
self.speculative_config is not None
|
||||
@@ -869,7 +869,7 @@ class GPUModelRunner(
|
||||
self._num_valid_draft_tokens_cpu = torch.empty(
|
||||
self.max_num_reqs, dtype=torch.int32, pin_memory=PIN_MEMORY
|
||||
)
|
||||
self._num_valid_draft_tokens_event = torch.Event()
|
||||
self._num_valid_draft_tokens_event = torch.cuda.Event()
|
||||
self._num_valid_draft_tokens_copy_stream = torch.cuda.Stream()
|
||||
|
||||
self._draft_token_req_ids: list[str] | None = None
|
||||
|
||||
@@ -49,6 +49,7 @@ def _torch_cuda_wrapper():
|
||||
torch.cuda.current_stream = partial(torch.xpu.current_stream)
|
||||
torch.cuda.stream = partial(torch.xpu.stream)
|
||||
torch.cuda.set_stream = partial(torch.xpu.set_stream)
|
||||
torch.cuda.Event = partial(torch.xpu.Event)
|
||||
if supports_xpu_graph():
|
||||
torch.cuda.graph = partial(torch.xpu.graph)
|
||||
torch.cuda.CUDAGraph = torch.xpu.XPUGraph
|
||||
|
||||
Reference in New Issue
Block a user