support InternLM

Former-commit-id: a2f507c56238d7fb2670edbab52d6b275f245e27

README.md

@@ -9,6 +9,8 @@
 
 ## Changelog
 
+[23/07/07] Now we support training the InternLM-7B model in this repo. Try `--model_name_or_path internlm/internlm-7b` argument to use the InternLM model. Remember to use `--prompt_template intern` argument when using the InternLM-chat model.
+
 [23/07/05] Now we support training the Falcon-7B/40B models in this repo. Try `--model_name_or_path tiiuae/falcon-7b` and `--lora_target query_key_value` arguments to use the Falcon model.
 
 [23/06/29] We provide a reproducible example of training a chat model using instruction-following datasets, see this [HuggingFace Repo](https://huggingface.co/hiyouga/baichuan-7b-sft) for details.

src/utils/seq2seq.py

@@ -104,8 +104,8 @@ class Seq2SeqPeftTrainer(PeftTrainer):
         preds = np.where(predict_results.predictions != IGNORE_INDEX, predict_results.predictions, self.tokenizer.pad_token_id)
         labels = np.where(predict_results.label_ids != IGNORE_INDEX, predict_results.label_ids, self.tokenizer.pad_token_id)
 
-        decoded_preds = self.tokenizer.batch_decode(preds, skip_special_tokens=True)
-        decoded_labels = self.tokenizer.batch_decode(labels, skip_special_tokens=True)
+        decoded_preds = self.tokenizer.batch_decode(preds, skip_special_tokens=True, clean_up_tokenization_spaces=True)
+        decoded_labels = self.tokenizer.batch_decode(labels, skip_special_tokens=True, clean_up_tokenization_spaces=True)
 
         with open(output_prediction_file, "w", encoding="utf-8") as writer:
             res: List[str] = []

src/utils/template.py

@@ -114,6 +114,17 @@ class Template:
                 use_history=True
             )
 
+        elif self.name == "intern":
+            r"""
+            Supports: https://huggingface.co/internlm/internlm-chat-7b
+            """
+            self._register_template(
+                prefix="",
+                prompt="<|User|>:{query}<eoh>\n<|Bot|>:",
+                sep="<eoa>\n",
+                use_history=True
+            )
+
         else:
             raise ValueError("Template {} does not exist.".format(self.name))