[Fix Bug]num_active_loras always equals to zero (#34119)
Signed-off-by: Runkai Tao <rt572@physics.rutgers.edu> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -187,7 +187,8 @@ def use_fused_moe_lora_kernel(
|
||||
|
||||
# num_active_loras is the number of active LoRAs
|
||||
# (max_loras + 1 to include no-lora case)
|
||||
num_active_loras = max_loras + 1
|
||||
# Stored as CPU tensor to match the kernel API (torch.compile compatibility)
|
||||
num_active_loras = torch.tensor([max_loras + 1], dtype=torch.int32, device="cpu")
|
||||
|
||||
fused_moe_lora(
|
||||
output,
|
||||
@@ -399,7 +400,8 @@ def use_fused_moe_lora_kernel_naive(
|
||||
|
||||
# num_active_loras is the number of active LoRAs
|
||||
# (max_loras + 1 to include no-lora case)
|
||||
num_active_loras = max_loras + 1
|
||||
# Stored as CPU tensor to match the kernel API (torch.compile compatibility)
|
||||
num_active_loras = torch.tensor([max_loras + 1], dtype=torch.int32, device="cpu")
|
||||
|
||||
fused_moe_lora(
|
||||
output,
|
||||
|
||||
@@ -70,8 +70,12 @@ def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
|
||||
@pytest.mark.parametrize("specialize_active_lora", [True, False])
|
||||
def test_gpt_oss_lora(
|
||||
monkeypatch: pytest.MonkeyPatch, gptoss20b_lora_files, mxfp4_use_marlin
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
gptoss20b_lora_files,
|
||||
mxfp4_use_marlin,
|
||||
specialize_active_lora,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0")
|
||||
@@ -83,6 +87,7 @@ def test_gpt_oss_lora(
|
||||
max_lora_rank=8,
|
||||
max_num_seqs=2,
|
||||
max_num_batched_tokens=2048,
|
||||
specialize_active_lora=specialize_active_lora,
|
||||
compilation_config=vllm.config.CompilationConfig( # Avoid OOM
|
||||
cudagraph_specialize_lora=False,
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user