From c083708433328915f92189c6cee36c44e0454f7f Mon Sep 17 00:00:00 2001 From: marko1616 Date: Sun, 24 Mar 2024 02:44:35 +0800 Subject: [PATCH] fix Llama lora merge crash Former-commit-id: c1e2c4ea45ad210e776a192e05e226b34d764135 --- src/llmtuner/train/tuner.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/llmtuner/train/tuner.py b/src/llmtuner/train/tuner.py index 1b8e3cb7..e9b195de 100644 --- a/src/llmtuner/train/tuner.py +++ b/src/llmtuner/train/tuner.py @@ -64,6 +64,15 @@ def export_model(args: Optional[Dict[str, Any]] = None): for param in model.parameters(): param.data = param.data.to(output_dtype) + # Configuration check and fix + config = model.generation_config + if config.do_sample and ( + (config.temperature is not None and config.temperature != 1.0) or + (config.top_p is not None and config.top_p != 1.0) or + (config.typical_p is not None and config.typical_p != 1.0) + ): + config.do_sample = False + model.save_pretrained( save_directory=model_args.export_dir, max_shard_size="{}GB".format(model_args.export_size),