From edeed5566468a9a91cf208bea6f666ac9d06473a Mon Sep 17 00:00:00 2001 From: marko1616 Date: Sun, 24 Mar 2024 02:44:35 +0800 Subject: [PATCH 1/4] fix Llama lora merge crash Former-commit-id: c29a2893f58cf7a916ff05b2725fadf1ad2c4c9a --- src/llmtuner/train/tuner.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/llmtuner/train/tuner.py b/src/llmtuner/train/tuner.py index 1b8e3cb7..e9b195de 100644 --- a/src/llmtuner/train/tuner.py +++ b/src/llmtuner/train/tuner.py @@ -64,6 +64,15 @@ def export_model(args: Optional[Dict[str, Any]] = None): for param in model.parameters(): param.data = param.data.to(output_dtype) + # Configuration check and fix + config = model.generation_config + if config.do_sample and ( + (config.temperature is not None and config.temperature != 1.0) or + (config.top_p is not None and config.top_p != 1.0) or + (config.typical_p is not None and config.typical_p != 1.0) + ): + config.do_sample = False + model.save_pretrained( save_directory=model_args.export_dir, max_shard_size="{}GB".format(model_args.export_size), From 2eba1c6851ea789ddaf567b303bdb3b190adffa2 Mon Sep 17 00:00:00 2001 From: marko1616 Date: Sun, 24 Mar 2024 02:55:23 +0800 Subject: [PATCH 2/4] fix Llama lora merge crash Former-commit-id: a8bd8e9149ff79a2707fec9c6d006761cfdd0dee --- src/llmtuner/train/tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llmtuner/train/tuner.py b/src/llmtuner/train/tuner.py index e9b195de..cd532f5e 100644 --- a/src/llmtuner/train/tuner.py +++ b/src/llmtuner/train/tuner.py @@ -71,7 +71,7 @@ def export_model(args: Optional[Dict[str, Any]] = None): (config.top_p is not None and config.top_p != 1.0) or (config.typical_p is not None and config.typical_p != 1.0) ): - config.do_sample = False + config.do_sample = True model.save_pretrained( save_directory=model_args.export_dir, From c555b2cce30942d88d423bea551dd867b5d0a920 Mon Sep 17 00:00:00 2001 From: marko1616 Date: Sun, 24 Mar 2024 03:06:11 +0800 Subject: [PATCH 3/4] fix Llama lora merge crash Former-commit-id: 46f7d8e6b85f73fb0c51c8b08bd9955c3b171d93 --- src/llmtuner/train/tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llmtuner/train/tuner.py b/src/llmtuner/train/tuner.py index cd532f5e..8eda9676 100644 --- a/src/llmtuner/train/tuner.py +++ b/src/llmtuner/train/tuner.py @@ -66,7 +66,7 @@ def export_model(args: Optional[Dict[str, Any]] = None): # Configuration check and fix config = model.generation_config - if config.do_sample and ( + if config.do_sample == False and ( (config.temperature is not None and config.temperature != 1.0) or (config.top_p is not None and config.top_p != 1.0) or (config.typical_p is not None and config.typical_p != 1.0) From 7f99cb1817b9cba873e0813bfee27fa03cb6cbb9 Mon Sep 17 00:00:00 2001 From: marko1616 Date: Sun, 24 Mar 2024 16:12:10 +0800 Subject: [PATCH 4/4] pass ruff check Former-commit-id: 8534b069a05121eb041371a6becccf0a1a23f268 --- src/llmtuner/train/tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llmtuner/train/tuner.py b/src/llmtuner/train/tuner.py index 8eda9676..9074572a 100644 --- a/src/llmtuner/train/tuner.py +++ b/src/llmtuner/train/tuner.py @@ -66,7 +66,7 @@ def export_model(args: Optional[Dict[str, Any]] = None): # Configuration check and fix config = model.generation_config - if config.do_sample == False and ( + if not config.do_sample and ( (config.temperature is not None and config.temperature != 1.0) or (config.top_p is not None and config.top_p != 1.0) or (config.typical_p is not None and config.typical_p != 1.0)