update data processors

This commit is contained in:
hiyouga
2024-06-07 04:15:40 +08:00
parent 181dbb0d05
commit ccc8b64cc2
6 changed files with 190 additions and 139 deletions

View File

@@ -72,7 +72,7 @@ def main():
elif command == Command.EXPORT:
export_model()
elif command == Command.TRAIN:
disable_torchrun = os.environ.get("DISABLE_TORCHRUN", "0").lower() in ["true", "1"]
disable_torchrun = os.environ.get("TORCHRUN_DISABLED", "0").lower() in ["true", "1"]
if disable_torchrun and get_device_count() > 1:
logger.warning("`torchrun` cannot be disabled when device count > 1.")
disable_torchrun = False