From 3553e301dd078973f62df8a1bbffe11c2823b09a Mon Sep 17 00:00:00 2001
From: hiyouga <hiyouga@buaa.edu.cn>
Date: Tue, 5 Mar 2024 11:51:41 +0800
Subject: [PATCH] fix dora inference

Former-commit-id: 21b3597b0a05169afe51e1609b532787a65ca8ea
---
 src/llmtuner/model/patcher.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/llmtuner/model/patcher.py b/src/llmtuner/model/patcher.py
index e7ff0486..5a79387c 100644
--- a/src/llmtuner/model/patcher.py
+++ b/src/llmtuner/model/patcher.py
@@ -287,6 +287,8 @@ def patch_config(
         init_kwargs["low_cpu_mem_usage"] = True
         if is_trainable:
             init_kwargs["device_map"] = {"": get_current_device()}
+        elif model_args.export_dir is None:
+            init_kwargs["device_map"] = "auto"
 
 
 def patch_model(