38 lines
1.8 KiB
Diff
38 lines
1.8 KiB
Diff
From 9d428fa5e8b11873e5af5452cbad76dee8715de1 Mon Sep 17 00:00:00 2001
|
|
From: Hinata <hinata@idihore.id>
|
|
Date: Sat, 10 Oct 2026 17:31:17 +0000
|
|
Subject: [PATCH 101/103] auto_gptq: skip the Marlin workspace when CUDA is not
|
|
initialized
|
|
|
|
The PLE offload worker builds weights with CUDA forbidden; its guard
|
|
(vllm/v1/ple_offload/worker.py) refuses torch.cuda lazy init. AutoGPTQ
|
|
create_weights calls marlin_make_workspace_new for modular MoE experts,
|
|
which initializes CUDA and kills the offload process at load time.
|
|
Marlin creates the workspace lazily at apply time when the layer has
|
|
none, so the eager workspace is only needed where CUDA is live. Skip it
|
|
everywhere else; native AutoRound checkpoints then load in the CUDA-free
|
|
worker.
|
|
---
|
|
vllm/model_executor/layers/quantization/auto_gptq.py | 5 ++++-
|
|
1 file changed, 4 insertions(+), 1 deletion(-)
|
|
|
|
diff --git a/vllm/model_executor/layers/quantization/auto_gptq.py b/vllm/model_executor/layers/quantization/auto_gptq.py
|
|
index 893f85b..770b25e 100644
|
|
--- a/vllm/model_executor/layers/quantization/auto_gptq.py
|
|
+++ b/vllm/model_executor/layers/quantization/auto_gptq.py
|
|
@@ -604,7 +604,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
|
self.experts_cls, FusedMoEExpertsModular
|
|
):
|
|
device = layer.w13_qweight.device
|
|
- layer.workspace = marlin_make_workspace_new(device, 4)
|
|
+ # PLE offload workers run CUDA-free; the Marlin kernel creates the
|
|
+ # workspace lazily at apply time when it is missing.
|
|
+ if device.type == "cuda" and torch.cuda.is_initialized():
|
|
+ layer.workspace = marlin_make_workspace_new(device, 4)
|
|
|
|
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
|
def replace_or_register(name: str, val: torch.Tensor | None):
|
|
--
|
|
2.43.0
|
|
|