[Model Runner V2] Fix lora Triton Error [CUDA]: device-side assert triggered (#43139)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -37,6 +37,7 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.layers import LoRAMapping
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
initialize_mamba_ssu_backend,
|
||||
)
|
||||
@@ -1084,7 +1085,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
block_tables = None
|
||||
slot_mappings = None
|
||||
# FIXME(woosuk): Fix warmup for LoRA.
|
||||
if self.lora_config:
|
||||
# program a no-LoRA mapping here so kernels early-exit instead of
|
||||
# reading uninitialized metadata during dummy runs.
|
||||
# FIXME: Replace this with LoRA warmup:
|
||||
# https://github.com/vllm-project/vllm/pull/35536
|
||||
assert hasattr(self, "lora_manager")
|
||||
self.lora_manager._adapter_manager.set_adapter_mapping(
|
||||
LoRAMapping(
|
||||
index_mapping=(0,) * input_batch.num_tokens_after_padding,
|
||||
prompt_mapping=(0,) * input_batch.num_reqs,
|
||||
is_prefill=True,
|
||||
)
|
||||
)
|
||||
|
||||
attn_metadata = None
|
||||
slot_mappings_by_layer = None
|
||||
|
||||
Reference in New Issue
Block a user