[V0 deprecation] Remove more V0 references (#29088)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -60,18 +60,9 @@ def llama_3p2_1b_files():
|
||||
|
||||
def _run_writer(input_dir, output_dir, weights_patterns, **kwargs):
|
||||
llm_sharded_writer = LLM(model=input_dir, **kwargs)
|
||||
# Check which engine version is being used
|
||||
is_v1_engine = hasattr(llm_sharded_writer.llm_engine, "engine_core")
|
||||
|
||||
# Dump worker states to output directory
|
||||
if is_v1_engine:
|
||||
# For V1 engine, we need to use engine_core.save_sharded_state
|
||||
print("Using V1 engine save path")
|
||||
llm_sharded_writer.llm_engine.engine_core.save_sharded_state(path=output_dir)
|
||||
else:
|
||||
# For V0 engine
|
||||
print("Using V0 engine save path")
|
||||
model_executor = llm_sharded_writer.llm_engine.model_executor
|
||||
model_executor.save_sharded_state(path=output_dir)
|
||||
llm_sharded_writer.llm_engine.engine_core.save_sharded_state(path=output_dir)
|
||||
|
||||
# Copy metadata files to output directory
|
||||
for file in os.listdir(input_dir):
|
||||
|
||||
+13
-12
@@ -140,21 +140,22 @@ CONFIGS: dict[str, ServerConfig] = {
|
||||
"without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
|
||||
"to the user's question - just respond to it normally.",
|
||||
},
|
||||
# V1 Test: Passing locally but failing in CI. This runs the
|
||||
# V0 Engine because of CPU offloading. Need to debug why.
|
||||
# FIXME: This test currently fails, need to debug why.
|
||||
# "granite20b": {
|
||||
# "model":
|
||||
# "mbayser/granite-20b-functioncalling-FP8-KV",
|
||||
# "model": "mbayser/granite-20b-functioncalling-FP8-KV",
|
||||
# "arguments": [
|
||||
# "--tool-call-parser", "granite-20b-fc", "--chat-template",
|
||||
# str(VLLM_PATH /
|
||||
# "examples/tool_chat_template_granite_20b_fc.jinja"),
|
||||
# "--max_num_seqs", "1", "--enforce-eager", "--cpu-offload-gb", "20"
|
||||
# "--tool-call-parser",
|
||||
# "granite-20b-fc",
|
||||
# "--chat-template",
|
||||
# str(VLLM_PATH / "examples/tool_chat_template_granite_20b_fc.jinja"),
|
||||
# "--max_num_seqs",
|
||||
# "1",
|
||||
# "--enforce-eager",
|
||||
# "--cpu-offload-gb",
|
||||
# "20",
|
||||
# ],
|
||||
# "supports_parallel":
|
||||
# False,
|
||||
# "supports_rocm":
|
||||
# False,
|
||||
# "supports_parallel": False,
|
||||
# "supports_rocm": False,
|
||||
# },
|
||||
"granite-3.0-8b": {
|
||||
"model": "ibm-granite/granite-3.0-8b-instruct",
|
||||
|
||||
Reference in New Issue
Block a user