From 55815ab1ff7c5220a73db7024c6cc940086af711 Mon Sep 17 00:00:00 2001 From: MengqingCao Date: Wed, 5 Jun 2024 08:03:30 +0000 Subject: [PATCH] fix #4077 Former-commit-id: 90ed3cae92c624cb53e7349bb7edeaf402d030e6 --- src/llamafactory/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/llamafactory/cli.py b/src/llamafactory/cli.py index c14ae6ec..8a229a38 100644 --- a/src/llamafactory/cli.py +++ b/src/llamafactory/cli.py @@ -70,7 +70,11 @@ def main(): elif command == Command.EXPORT: export_model() elif command == Command.TRAIN: - if get_device_count() > 1: + if get_device_count() > 0: + # NOTE (MengqingCao): why use torchrun when only one accelerator is available? + # DeepSpeed only warp model with DeepSpeedEngine when launching by distributed launcher, + # e.g., torchrun, causing some feature missing + # sa: https://github.com/huggingface/transformers/issues/24309 master_addr = os.environ.get("MASTER_ADDR", "127.0.0.1") master_port = os.environ.get("MASTER_PORT", str(random.randint(20001, 29999))) logger.info("Initializing distributed tasks at: {}:{}".format(master_addr, master_port))