From 77f6647e8f6f4ac373e3593886c64e4ad4370e41 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Mon, 7 Aug 2023 13:34:35 +0800 Subject: [PATCH] update trainer Former-commit-id: 220175ab2410ce22a553344eb75d5a556ed1a276 --- src/llmtuner/tuner/core/trainer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/llmtuner/tuner/core/trainer.py b/src/llmtuner/tuner/core/trainer.py index 9541805f..ae80f32f 100644 --- a/src/llmtuner/tuner/core/trainer.py +++ b/src/llmtuner/tuner/core/trainer.py @@ -67,7 +67,10 @@ class PeftTrainer(Seq2SeqTrainer): torch.save(state_dict, os.path.join(output_dir, WEIGHTS_NAME)) if self.finetuning_args.finetuning_type == "full" and self.tokenizer is not None: - self.tokenizer.save_pretrained(output_dir) + try: + self.tokenizer.save_pretrained(output_dir) + except: + logger.warning("Cannot save tokenizer, copy the files manually.") with open(os.path.join(output_dir, TRAINING_ARGS_NAME), "w", encoding="utf-8") as f: f.write(self.args.to_json_string() + "\n")