From 82a79e9fdfd3e77647f87d5b13c3d32e1efd4a4e Mon Sep 17 00:00:00 2001
From: hiyouga <hiyouga@buaa.edu.cn>
Date: Wed, 20 Dec 2023 17:15:40 +0800
Subject: [PATCH] fix #1073 #1462 #1735 #1908

Former-commit-id: 31165a9822bd52130b33cd3439f887c26e0679dc
---
 src/llmtuner/train/sft/workflow.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/llmtuner/train/sft/workflow.py b/src/llmtuner/train/sft/workflow.py
index 94a81151..0e9bf7e4 100644
--- a/src/llmtuner/train/sft/workflow.py
+++ b/src/llmtuner/train/sft/workflow.py
@@ -32,6 +32,9 @@ def run_sft(
     if training_args.predict_with_generate:
         tokenizer.padding_side = "left" # use left-padding in generation
 
+    if getattr(model, "is_quantized", False) and not training_args.do_train:
+        setattr(model, "_hf_peft_config_loaded", True) # hack here: make model compatible with prediction
+
     data_collator = DataCollatorForSeq2Seq(
         tokenizer=tokenizer,
         pad_to_multiple_of=8 if tokenizer.padding_side == "right" else None, # for shift short attention