[XPU] Enable XPU eplb (#44987)
Signed-off-by: mayuyuace <qiming1.zhang@intel.com> Signed-off-by: Qiming Zhang <qiming1.zhang@intel.com>
This commit is contained in:
@@ -13,6 +13,7 @@ from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
mp.set_start_method("spawn", force=True)
|
||||
@@ -73,13 +74,16 @@ def distributed_run(fn, world_size, *args):
|
||||
def set_env_vars_and_device(env: dict[str, str]) -> None:
|
||||
update_environment_variables(env)
|
||||
local_rank = os.environ["LOCAL_RANK"]
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
backend = "xccl" if current_platform.is_xpu() else "nccl"
|
||||
|
||||
# Create a minimal vllm config for init_distributed_environment
|
||||
vllm_config = VllmConfig()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
init_distributed_environment()
|
||||
init_distributed_environment(backend=backend)
|
||||
atexit.register(_destroy_process_group_if_initialized)
|
||||
# Ensure each worker process has the same random seed
|
||||
random.seed(42)
|
||||
|
||||
@@ -10,6 +10,8 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..evals.gsm8k.gsm8k_eval import evaluate_gsm8k
|
||||
from ..utils import RemoteOpenAIServer, multi_gpu_test
|
||||
|
||||
@@ -233,6 +235,8 @@ def test_elastic_ep_scaling(
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("Elastic EP with NIXL is not supported on XPU")
|
||||
if all2all_backend == "nixl_ep" and not has_nixl_ep():
|
||||
pytest.skip("NIXL EP is not installed")
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.distributed.eplb.eplb_state import (
|
||||
compute_logical_maps,
|
||||
)
|
||||
from vllm.distributed.eplb.policy.default import DefaultEplbPolicy
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_eplb_load_stats_reduce_across_ranks():
|
||||
@@ -263,11 +264,13 @@ def test_global_load_balance_fallback():
|
||||
assert torch.sum(logcnt) == num_replicas
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda", "xpu"])
|
||||
def test_device_compatibility(device):
|
||||
"""Test device compatibility."""
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
if device == "xpu" and not current_platform.is_xpu():
|
||||
pytest.skip("XPU not available")
|
||||
|
||||
weight = torch.tensor([[10, 20, 30, 40]], device=device)
|
||||
num_replicas = 6
|
||||
|
||||
@@ -4,15 +4,17 @@
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent
|
||||
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_wait_blocks_until_record():
|
||||
event = CpuGpuEvent()
|
||||
record_stream = torch.cuda.Stream()
|
||||
wait_stream = torch.cuda.Stream()
|
||||
record_stream = torch.Stream()
|
||||
wait_stream = torch.Stream()
|
||||
wait_returned = threading.Event()
|
||||
|
||||
def waiter():
|
||||
@@ -33,8 +35,8 @@ def test_wait_blocks_until_record():
|
||||
|
||||
def test_reuse_across_multiple_cycles():
|
||||
wrapper = CpuGpuEvent()
|
||||
record_stream = torch.cuda.Stream()
|
||||
wait_stream = torch.cuda.Stream()
|
||||
record_stream = torch.Stream()
|
||||
wait_stream = torch.Stream()
|
||||
NUM_CYCLES = 8
|
||||
completed_cycles = []
|
||||
barriers = [threading.Barrier(2) for _ in range(NUM_CYCLES)]
|
||||
@@ -56,13 +58,18 @@ def test_reuse_across_multiple_cycles():
|
||||
assert len(completed_cycles) == NUM_CYCLES
|
||||
|
||||
|
||||
def test_producer_consumer():
|
||||
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared
|
||||
GPU tensor on multiple CPU threads.
|
||||
@pytest.mark.parametrize("device", ["cuda", "xpu"])
|
||||
def test_producer_consumer(device):
|
||||
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from
|
||||
a shared GPU tensor on multiple CPU threads.
|
||||
"""
|
||||
worker_stream = torch.cuda.Stream()
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
if device == "xpu" and not current_platform.is_xpu():
|
||||
pytest.skip("XPU not available")
|
||||
worker_stream = torch.Stream()
|
||||
# Create a single element counter that will be shared between two threads
|
||||
buf = torch.zeros(1, device="cuda")
|
||||
buf = torch.zeros(1, device=device)
|
||||
NUM_ROUNDS = 5
|
||||
|
||||
ready_cpu = [threading.Event() for _ in range(NUM_ROUNDS)]
|
||||
@@ -76,7 +83,7 @@ def test_producer_consumer():
|
||||
if i > 0:
|
||||
events[i - 1].wait(stream=worker_stream)
|
||||
|
||||
with torch.cuda.stream(worker_stream):
|
||||
with device_stream(worker_stream):
|
||||
buf.fill_(float(i + 1))
|
||||
|
||||
worker_stream.synchronize()
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
@@ -208,10 +209,14 @@ def verify_redundant_experts_have_same_weights(
|
||||
ep_rank: int,
|
||||
world_size: int,
|
||||
num_local_experts: int,
|
||||
cpu_group,
|
||||
) -> bool:
|
||||
"""Verify that all replicas of the same logical expert have the same weights."""
|
||||
num_layers = len(expert_weights)
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
# XPU: gloo does not support XPU tensors, so gather on CPU then move back.
|
||||
# CUDA: gloo supports CUDA tensors directly, keep original device.
|
||||
use_cpu_gather = current_platform.is_xpu()
|
||||
|
||||
ok = True
|
||||
for layer in range(num_layers):
|
||||
@@ -219,31 +224,35 @@ def verify_redundant_experts_have_same_weights(
|
||||
all_weights: list[torch.Tensor] = []
|
||||
|
||||
for weight_idx, hidden_size in enumerate(hidden_sizes):
|
||||
orig_device = expert_weights[layer][weight_idx].device
|
||||
|
||||
if use_cpu_gather:
|
||||
local_weights = expert_weights[layer][weight_idx].cpu()
|
||||
gather_device = "cpu"
|
||||
else:
|
||||
local_weights = expert_weights[layer][weight_idx]
|
||||
gather_device = orig_device
|
||||
|
||||
# Create tensor to store all expert weights
|
||||
# Shape: [total_physical_experts, hidden_size]
|
||||
gathered_weights = torch.zeros(
|
||||
total_physical_experts,
|
||||
hidden_size,
|
||||
device=expert_weights[layer][weight_idx].device,
|
||||
dtype=expert_weights[layer][weight_idx].dtype,
|
||||
device=gather_device,
|
||||
dtype=local_weights.dtype,
|
||||
)
|
||||
|
||||
# Use all_gather to collect expert weights from current node
|
||||
# expert_weights[layer][weight_idx] shape:
|
||||
# [num_local_experts, hidden_size]
|
||||
local_weights = expert_weights[layer][
|
||||
weight_idx
|
||||
] # [num_local_experts, hidden_size]
|
||||
|
||||
# Split tensor along dim 0 into a list for all_gather
|
||||
gathered_weights_list = torch.chunk(gathered_weights, world_size, dim=0)
|
||||
|
||||
torch.distributed.all_gather(
|
||||
# Output list: each element corresponds to one rank's weights
|
||||
list(gathered_weights_list),
|
||||
local_weights, # Input: current rank's local weights
|
||||
local_weights,
|
||||
group=cpu_group,
|
||||
)
|
||||
|
||||
if use_cpu_gather:
|
||||
gathered_weights = gathered_weights.to(orig_device)
|
||||
all_weights.append(gathered_weights)
|
||||
|
||||
# Verify that all replicas of the same logical expert have the same
|
||||
@@ -294,9 +303,23 @@ def verify_redundant_experts_have_same_weights(
|
||||
return ok
|
||||
|
||||
|
||||
def assert_verification_synced(local_ok: bool, msg: str) -> None:
|
||||
ok_tensor = torch.tensor([1 if local_ok else 0], device="cuda", dtype=torch.int32)
|
||||
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
|
||||
def assert_verification_synced(local_ok: bool, msg: str, cpu_group) -> None:
|
||||
if current_platform.is_xpu():
|
||||
# XPU: default group backend (gloo) does not support XPU tensors;
|
||||
# use CPU tensor with cpu_group instead.
|
||||
ok_tensor = torch.tensor(
|
||||
[1 if local_ok else 0], device="cpu", dtype=torch.int32
|
||||
)
|
||||
torch.distributed.all_reduce(
|
||||
ok_tensor, op=torch.distributed.ReduceOp.MIN, group=cpu_group
|
||||
)
|
||||
else:
|
||||
# CUDA: nccl (default group) supports device tensors directly.
|
||||
device = torch.accelerator.current_accelerator()
|
||||
ok_tensor = torch.tensor(
|
||||
[1 if local_ok else 0], device=device, dtype=torch.int32
|
||||
)
|
||||
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
|
||||
assert bool(ok_tensor.item()), msg
|
||||
|
||||
|
||||
@@ -336,8 +359,10 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
hidden_sizes = [16, 32]
|
||||
@@ -376,7 +401,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
new_indices_cpu = new_indices.cpu()
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
cuda_stream = torch.cuda.Stream(device=device)
|
||||
stream = torch.Stream(device=device)
|
||||
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
@@ -384,7 +409,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
communicator.set_stream(cuda_stream)
|
||||
communicator.set_stream(stream)
|
||||
|
||||
for layer_idx in range(num_layers):
|
||||
transfer_metadata = transfer_layer(
|
||||
@@ -394,10 +419,10 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
expert_weights_buffer=expert_buffer,
|
||||
ep_group=ep_group,
|
||||
communicator=communicator,
|
||||
cuda_stream=cuda_stream,
|
||||
stream=stream,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
cuda_stream.synchronize()
|
||||
stream.synchronize()
|
||||
move_from_buffer(
|
||||
expert_weights=expert_weights[layer_idx],
|
||||
expert_weights_buffers=expert_buffer,
|
||||
@@ -421,6 +446,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
@@ -428,6 +454,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
local_ok,
|
||||
"Async transfer verification failed on at least one rank. "
|
||||
"See logs for details.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -452,9 +479,11 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
# Test parameters
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
@@ -523,12 +552,14 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Rearrange verification failed on at least one rank. See logs for details.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -556,7 +587,7 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"eplb_communicator", ["torch_nccl", "torch_gloo", "pynccl", "nixl"]
|
||||
"eplb_communicator", ["torch_nccl", "torch_gloo", "torch_xccl", "pynccl", "nixl"]
|
||||
)
|
||||
def test_rearrange_expert_weights_with_redundancy(
|
||||
world_size,
|
||||
@@ -568,6 +599,12 @@ def test_rearrange_expert_weights_with_redundancy(
|
||||
"""Test the functionality of rearranging expert weights with redundancy."""
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if eplb_communicator == "nixl" and current_platform.is_xpu():
|
||||
pytest.skip("NIXL does not support XPU")
|
||||
if eplb_communicator in ("torch_nccl", "pynccl") and not torch.cuda.is_available():
|
||||
pytest.skip(f"{eplb_communicator} requires CUDA")
|
||||
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
|
||||
pytest.skip("torch_xccl requires XPU")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
@@ -592,9 +629,11 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
num_layers = 2
|
||||
num_local_experts = 2
|
||||
@@ -623,9 +662,10 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
original_weights.append(layer_copy)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend="torch_nccl",
|
||||
backend=default_backend,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
@@ -657,6 +697,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"No-change EPLB verification failed on at least one rank.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -666,7 +707,7 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
(2, 2, 2, 3),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "nixl"])
|
||||
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "torch_xccl", "nixl"])
|
||||
def test_async_transfer_layer_without_mtp(
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
@@ -677,6 +718,10 @@ def test_async_transfer_layer_without_mtp(
|
||||
"""Exercise async EPLB transfer path without MTP/spec decode."""
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if eplb_communicator == "nixl" and current_platform.is_xpu():
|
||||
pytest.skip("NIXL does not support XPU")
|
||||
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
|
||||
pytest.skip("torch_xccl requires XPU")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
|
||||
@@ -715,9 +760,11 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.cpu_group
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
num_layers = 1
|
||||
num_local_experts = 2
|
||||
@@ -753,9 +800,10 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
original_weights.append(layer_copy)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend="torch_nccl",
|
||||
backend=default_backend,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
@@ -788,6 +836,7 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Profile-mode EPLB verification failed on at least one rank.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.distributed.parallel_state import (
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
@@ -81,7 +82,7 @@ def make_fused_moe_layer(
|
||||
)
|
||||
re = fml.routed_experts
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
device = torch.device(f"{torch.accelerator.current_accelerator().type}:{rank}")
|
||||
|
||||
from functools import partial
|
||||
|
||||
@@ -190,13 +191,14 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
vllm_config.parallel_config.enable_eplb = True
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group = get_tp_group().cpu_group
|
||||
ep_group = get_tp_group().device_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
|
||||
fml_layers = [
|
||||
@@ -216,9 +218,10 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
||||
shuffled_indices[lidx] = torch.randperm(test_config.num_experts)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator(
|
||||
group_coordinator=get_eplb_group(),
|
||||
backend="torch_nccl",
|
||||
backend=default_backend,
|
||||
expert_weights=rank_expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
ModelOptNvFp4FusedMoE,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
@@ -270,6 +271,8 @@ def test_eplb_fml(
|
||||
):
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("NVFP4 quantization and flashinfer backends are CUDA-only")
|
||||
|
||||
num_local_experts = num_experts // world_size
|
||||
num_topk = 4
|
||||
|
||||
Reference in New Issue
Block a user