[Perf][4/n] Eliminate various GPU<->CPU syncs (#42347)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -30,10 +30,8 @@ from vllm.model_executor.models.interfaces import supports_any_eagle
|
||||
from vllm.multimodal import NestedTensors
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.platform_utils import (
|
||||
is_pin_memory_available,
|
||||
)
|
||||
from vllm.utils.torch_utils import (
|
||||
async_tensor_h2d,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
|
||||
@@ -498,10 +496,9 @@ def isin_list(
|
||||
elements: torch.Tensor,
|
||||
test_elements_list: list[int],
|
||||
) -> torch.Tensor:
|
||||
test_elements = torch.tensor(
|
||||
test_elements_list,
|
||||
pin_memory=is_pin_memory_available(),
|
||||
).to(device=elements.device, non_blocking=True)
|
||||
test_elements = async_tensor_h2d(
|
||||
test_elements_list, dtype=torch.int64, device=elements.device
|
||||
)
|
||||
|
||||
return torch.isin(elements, test_elements)
|
||||
|
||||
@@ -772,14 +769,9 @@ def extract_layer_index(layer_name: str, num_attn_module: int = 1) -> int:
|
||||
return layer_index
|
||||
|
||||
|
||||
def cast_overflow_tensors(
|
||||
tensors: torch.Tensor,
|
||||
offset: float = 1000,
|
||||
) -> torch.Tensor:
|
||||
if tensors.isinf().any() or tensors.isnan().any():
|
||||
clamp_value = torch.finfo(tensors.dtype).max - offset
|
||||
tensors = torch.clamp(tensors, min=-clamp_value, max=clamp_value)
|
||||
return tensors
|
||||
def cast_overflow_tensors(tensors: torch.Tensor, offset: float = 1000) -> torch.Tensor:
|
||||
clamp_value = torch.finfo(tensors.dtype).max - offset
|
||||
return torch.clamp(tensors, min=-clamp_value, max=clamp_value)
|
||||
|
||||
|
||||
def fast_topk(
|
||||
|
||||
Reference in New Issue
Block a user